diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml index 4e88aef..4c3d953 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml @@ -29,7 +29,6 @@ Hadoop YARN REST APIs for services - false 1.6.5 @@ -81,30 +80,10 @@ org.apache.maven.plugins maven-surefire-plugin - ${maven-surefire-plugin.version} - ${test.reuseForks} - ${test.forkMode} - 1 - ${test.forkedProcessTimeoutInSeconds} - - 1 - ${test.argLine} - ${test.failIfNoTests} - ${build.redirect.test.output.to.file} - ${test.env.path} + ${java.home} - - true - true - - - **/Test*.java - - - **/Test*$*.java - diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/services/api/impl/TestApplicationApiService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/services/api/impl/TestApplicationApiService.java index 6e077d2..22debd7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/services/api/impl/TestApplicationApiService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/services/api/impl/TestApplicationApiService.java @@ -17,24 +17,10 @@ package org.apache.hadoop.yarn.services.api.impl; -import static org.apache.slider.util.RestApiConstants.*; -import static org.apache.slider.util.RestApiErrorMessages.*; - -import java.util.ArrayList; - -import org.apache.slider.api.resource.Application; -import org.apache.slider.api.resource.Artifact; -import org.apache.slider.api.resource.Resource; -import org.apache.slider.util.ServiceApiUtil; -import org.junit.After; -import org.junit.Assert; import org.junit.Before; -import org.junit.Test; import org.junit.runner.RunWith; import org.powermock.core.classloader.annotations.SuppressStaticInitializationFor; import org.powermock.modules.junit4.PowerMockRunner; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * Test class for application life time monitor feature test. @@ -42,168 +28,14 @@ @RunWith(PowerMockRunner.class) @SuppressStaticInitializationFor("org.apache.hadoop.yarn.services.api.impl.ApplicationApiService") public class TestApplicationApiService { - private static final Logger logger = LoggerFactory - .getLogger(TestApplicationApiService.class); - private static String EXCEPTION_PREFIX = "Should have thrown exception: "; - private static String NO_EXCEPTION_PREFIX = "Should not have thrown exception: "; private ApplicationApiService appApiService; @Before public void setup() throws Exception { - appApiService = new ApplicationApiService(); + appApiService = new ApplicationApiService(); } - @After - public void tearDown() throws Exception { - } - - @Test(timeout = 90000) - public void testValidateApplicationPostPayload() throws Exception { - Application app = new Application(); - - // no name - try { - ServiceApiUtil.validateApplicationPayload(app, null); - Assert.fail(EXCEPTION_PREFIX + "application with no name"); - } catch (IllegalArgumentException e) { - Assert.assertEquals(ERROR_APPLICATION_NAME_INVALID, e.getMessage()); - } - - // bad format name - String[] badNames = { "4finance", "Finance", "finance@home" }; - for (String badName : badNames) { - app.setName(badName); - try { - ServiceApiUtil.validateApplicationPayload(app, null); - Assert.fail(EXCEPTION_PREFIX + "application with bad name " + badName); - } catch (IllegalArgumentException e) { - Assert.assertEquals(ERROR_APPLICATION_NAME_INVALID_FORMAT, - e.getMessage()); - } - } - - // no artifact - app.setName("finance_home"); - try { - ServiceApiUtil.validateApplicationPayload(app, null); - Assert.fail(EXCEPTION_PREFIX + "application with no artifact"); - } catch (IllegalArgumentException e) { - Assert.assertEquals(ERROR_ARTIFACT_INVALID, e.getMessage()); - } - - // no artifact id - Artifact artifact = new Artifact(); - app.setArtifact(artifact); - try { - ServiceApiUtil.validateApplicationPayload(app, null); - Assert.fail(EXCEPTION_PREFIX + "application with no artifact id"); - } catch (IllegalArgumentException e) { - Assert.assertEquals(ERROR_ARTIFACT_ID_INVALID, e.getMessage()); - } - - // if artifact is of type APPLICATION then everything is valid here - artifact.setType(Artifact.TypeEnum.APPLICATION); - artifact.setId("app.io/hbase:facebook_0.2"); - app.setNumberOfContainers(5l); - try { - ServiceApiUtil.validateApplicationPayload(app, null); - } catch (IllegalArgumentException e) { - logger.error("application attributes specified should be valid here", e); - Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage()); - } - - // default-component, default-lifetime and the property component_type - // should get assigned here - Assert.assertEquals(app.getComponents().get(0).getName(), - DEFAULT_COMPONENT_NAME); - Assert.assertEquals(app.getLifetime(), DEFAULT_UNLIMITED_LIFETIME); - //TODO handle external app - - // unset artifact type, default component and no of containers to test other - // validation logic - artifact.setType(null); - app.setComponents(new ArrayList<>()); - app.setNumberOfContainers(null); - - // resource not specified - artifact.setId("docker.io/centos:centos7"); - try { - ServiceApiUtil.validateApplicationPayload(app, null); - Assert.fail(EXCEPTION_PREFIX + "application with no resource"); - } catch (IllegalArgumentException e) { - Assert.assertEquals(ERROR_RESOURCE_INVALID, e.getMessage()); - } - - // memory not specified - Resource res = new Resource(); - app.setResource(res); - try { - ServiceApiUtil.validateApplicationPayload(app, null); - Assert.fail(EXCEPTION_PREFIX + "application with no memory"); - } catch (IllegalArgumentException e) { - Assert.assertEquals(ERROR_RESOURCE_MEMORY_INVALID, e.getMessage()); - } - - // cpu does not need to be always specified, it's an optional feature in yarn - // invalid no of cpus - res.setMemory("100mb"); - res.setCpus(-2); - try { - ServiceApiUtil.validateApplicationPayload(app, null); - Assert.fail( - EXCEPTION_PREFIX + "application with invalid no of cpups"); - } catch (IllegalArgumentException e) { - Assert.assertEquals(ERROR_RESOURCE_CPUS_INVALID_RANGE, e.getMessage()); - } - - // number of containers not specified - res.setCpus(2); - try { - ServiceApiUtil.validateApplicationPayload(app, null); - Assert.fail( - EXCEPTION_PREFIX + "application with no container count"); - } catch (IllegalArgumentException e) { - Assert.assertTrue(e.getMessage().contains(ERROR_CONTAINERS_COUNT_INVALID)); - } - - // specifying profile along with cpus/memory raises exception - res.setProfile("hbase_finance_large"); - try { - ServiceApiUtil.validateApplicationPayload(app, null); - Assert.fail(EXCEPTION_PREFIX - + "application with resource profile along with cpus/memory"); - } catch (IllegalArgumentException e) { - Assert.assertEquals(ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_NOT_SUPPORTED, - e.getMessage()); - } - - // currently resource profile alone is not supported. - // TODO: remove the next test once it is supported. - res.setCpus(null); - res.setMemory(null); - try { - ServiceApiUtil.validateApplicationPayload(app, null); - Assert.fail(EXCEPTION_PREFIX - + "application with resource profile only - NOT SUPPORTED"); - } catch (IllegalArgumentException e) { - Assert.assertEquals(ERROR_RESOURCE_PROFILE_NOT_SUPPORTED_YET, - e.getMessage()); - } - - // unset profile here and add cpus/memory back - res.setProfile(null); - res.setCpus(2); - res.setMemory("2gb"); - - // everything valid here - app.setNumberOfContainers(5l); - try { - ServiceApiUtil.validateApplicationPayload(app, null); - } catch (IllegalArgumentException e) { - logger.error("application attributes specified should be valid here", e); - Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage()); - } - - // Now test with components + public void testServicesApi() throws Exception { + // TODO write test for API } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Component.java index e7f3796..f73c305 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Component.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Component.java @@ -50,9 +50,9 @@ private String name = null; private List dependencies = new ArrayList(); private ReadinessCheck readinessCheck = null; - private Artifact artifact = new Artifact(); + private Artifact artifact = null; private String launchCommand = null; - private Resource resource = new Resource(); + private Resource resource = null; private Long numberOfContainers = null; private Boolean uniqueComponentSupport = false; private Boolean runPrivilegedContainer = false; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java index 37d1a40..e89306c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java @@ -22,6 +22,7 @@ import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import org.apache.commons.lang.StringUtils; +import org.apache.slider.common.tools.SliderUtils; import java.io.Serializable; import java.util.ArrayList; @@ -197,8 +198,10 @@ private String toIndentedString(java.lang.Object o) { * this ConfigFile. */ public synchronized void mergeFrom(Configuration that) { - this.properties.putAll(that.getProperties()); - this.env.putAll(that.getEnv()); + SliderUtils.mergeMapsIgnoreDuplicateKeys(this.properties, that + .getProperties()); + SliderUtils.mergeMapsIgnoreDuplicateKeys(this.env, that.getEnv()); + Map thatMap = new HashMap<>(); for (ConfigFile file : that.getFiles()) { thatMap.put(file.getDestFile(), file.copy()); @@ -206,7 +209,8 @@ public synchronized void mergeFrom(Configuration that) { for (ConfigFile thisFile : files) { if(thatMap.containsKey(thisFile.getDestFile())) { ConfigFile thatFile = thatMap.get(thisFile.getDestFile()); - thisFile.getProps().putAll(thatFile.getProps()); + SliderUtils.mergeMapsIgnoreDuplicateKeys(thisFile.getProps(), + thatFile.getProps()); thatMap.remove(thisFile.getDestFile()); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java index 32d78b4..5174d9a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -101,6 +101,7 @@ import org.apache.slider.common.params.ClientArgs; import org.apache.slider.common.params.CommonArgs; import org.apache.slider.common.tools.ConfigHelper; +import org.apache.slider.common.tools.Duration; import org.apache.slider.common.tools.SliderFileSystem; import org.apache.slider.common.tools.SliderUtils; import org.apache.slider.common.tools.SliderVersionInfo; @@ -636,16 +637,17 @@ private Application getApplicationFromArgs(String clusterName, public int actionBuild(Application application) throws YarnException, IOException { Path appDir = checkAppNotExistOnHdfs(application); + ServiceApiUtil.validateApplicationPayload(application, sliderFileSystem); persistApp(appDir, application); + deployedClusterName = application.getName(); return EXIT_SUCCESS; } public ApplicationId actionCreate(Application application) throws IOException, YarnException { - ServiceApiUtil.validateApplicationPayload(application, - sliderFileSystem.getFileSystem()); String appName = application.getName(); validateClusterName(appName); + ServiceApiUtil.validateApplicationPayload(application, sliderFileSystem); verifyNoLiveApp(appName, "Create"); Path appDir = checkAppNotExistOnHdfs(application); @@ -880,6 +882,14 @@ private Path checkAppNotExistOnHdfs(Application application) return appDir; } + private Path checkAppExistOnHdfs(String appName) + throws IOException, SliderException { + Path appDir = sliderFileSystem.buildClusterDirPath(appName); + sliderFileSystem.verifyPathExists( + new Path(appDir, appName + ".json")); + return appDir; + } + private void persistApp(Path appDir, Application application) throws IOException, SliderException { FsPermission appDirPermission = new FsPermission("750"); @@ -1165,19 +1175,20 @@ private int doClientInstall(ActionClientArgs clientInfo) require(pkgFile.isFile(), E_UNABLE_TO_READ_SUPPLIED_PACKAGE_FILE + " at %s", pkgFile.getAbsolutePath()); - JSONObject config = null; - if(clientInfo.clientConfig != null) { - try { - byte[] encoded = Files.toByteArray(clientInfo.clientConfig); - config = new JSONObject(new String(encoded, Charset.defaultCharset())); - } catch (JSONException jsonEx) { - log.error("Unable to read supplied configuration at {}: {}", - clientInfo.clientConfig, jsonEx); - log.debug("Unable to read supplied configuration at {}: {}", - clientInfo.clientConfig, jsonEx, jsonEx); - throw new BadConfigException(E_MUST_BE_A_VALID_JSON_FILE, jsonEx); - } - } + // JSONObject config = null; + // if(clientInfo.clientConfig != null) { + // try { + // byte[] encoded = Files.toByteArray(clientInfo.clientConfig); + // config = new JSONObject( + // new String(encoded, Charset.defaultCharset())); + // } catch (JSONException jsonEx) { + // log.error("Unable to read supplied configuration at {}: {}", + // clientInfo.clientConfig, jsonEx); + // log.debug("Unable to read supplied configuration at {}: {}", + // clientInfo.clientConfig, jsonEx, jsonEx); + // throw new BadConfigException(E_MUST_BE_A_VALID_JSON_FILE, jsonEx); + // } + // } // TODO handle client install // Only INSTALL is supported @@ -1802,23 +1813,24 @@ public int actionStop(String appName, ActionFreezeArgs freezeArgs) public int actionStart(String appName, ActionThawArgs thaw) throws YarnException, IOException { validateClusterName(appName); + Path appDir = checkAppExistOnHdfs(appName); + Application application = ServiceApiUtil.loadApplication(sliderFileSystem, + appName); + ServiceApiUtil.validateApplicationPayload(application, sliderFileSystem); // see if it is actually running and bail out; verifyNoLiveApp(appName, "Thaw"); - Path appDir = sliderFileSystem.buildClusterDirPath(appName); - Path appJson = new Path(appDir, appName + ".json"); - Application application = - jsonSerDeser.load(sliderFileSystem.getFileSystem(), appJson); - submitApp(application); + ApplicationId appId = submitApp(application); + application.setId(appId.toString()); + // write app definition on to hdfs + persistApp(appDir, application); return 0; } public Map flex(String appName, Map componentCounts) throws YarnException, IOException { validateClusterName(appName); - Path appDir = sliderFileSystem.buildClusterDirPath(appName); - Path appJson = new Path(appDir, appName + ".json"); - Application persistedApp = - jsonSerDeser.load(sliderFileSystem.getFileSystem(), appJson); + Application persistedApp = ServiceApiUtil.loadApplication(sliderFileSystem, + appName); Map original = new HashMap<>(componentCounts.size()); for (Component persistedComp : persistedApp.getComponents()) { String name = persistedComp.getName(); @@ -1833,7 +1845,8 @@ public int actionStart(String appName, ActionThawArgs thaw) + " do not exist in app definition."); } jsonSerDeser - .save(sliderFileSystem.getFileSystem(), appJson, persistedApp, true); + .save(sliderFileSystem.getFileSystem(), ServiceApiUtil.getAppJsonPath( + sliderFileSystem, appName), persistedApp, true); log.info("Updated app definition file for components " + componentCounts .keySet()); @@ -2705,6 +2718,18 @@ private int actionTokens(ActionTokensArgs args) yarnClient); } + @VisibleForTesting + public ApplicationReport monitorAppToRunning(Duration duration) + throws YarnException, IOException { + return monitorAppToState(YarnApplicationState.RUNNING, duration); + } + + @VisibleForTesting + public ApplicationReport monitorAppToState(YarnApplicationState desiredState, + Duration duration) + throws YarnException, IOException { + return yarnClient.monitorAppToState(applicationId, desiredState, duration); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java index 4839395..306bd99 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java @@ -245,4 +245,65 @@ public NodeInformationList listNodes(String label, boolean live) } return results; } + + /** + * Monitor the submitted application for reaching the requested state. + * Will also report if the app reaches a later state (failed, killed, etc) + * Kill application if duration!= null & time expires. + * @param appId Application Id of application to be monitored + * @param duration how long to wait -must be more than 0 + * @param desiredState desired state. + * @return the application report -null on a timeout + * @throws YarnException + * @throws IOException + */ + public ApplicationReport monitorAppToState( + ApplicationId appId, YarnApplicationState desiredState, Duration duration) + throws YarnException, IOException { + + if (appId == null) { + throw new BadCommandArgumentsException("null application ID"); + } + if (duration.limit <= 0) { + throw new BadCommandArgumentsException("Invalid monitoring duration"); + } + log.debug("Waiting {} millis for app to reach state {} ", + duration.limit, + desiredState); + duration.start(); + try { + while (true) { + // Get application report for the appId we are interested in + + ApplicationReport r = getApplicationReport(appId); + + log.debug("queried status is\n{}", + new SliderUtils.OnDemandReportStringifier(r)); + + YarnApplicationState state = r.getYarnApplicationState(); + if (state.ordinal() >= desiredState.ordinal()) { + log.debug("App in desired state (or higher) :{}", state); + return r; + } + if (duration.getLimitExceeded()) { + log.debug( + "Wait limit of {} millis to get to state {}, exceeded; app " + + "status\n {}", + duration.limit, + desiredState, + new SliderUtils.OnDemandReportStringifier(r)); + return null; + } + + // sleep 1s. + try { + Thread.sleep(1000); + } catch (InterruptedException ignored) { + log.debug("Thread sleep in monitoring loop interrupted"); + } + } + } finally { + duration.close(); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java index bc8e139..6dc51ec 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java @@ -48,8 +48,6 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.slider.Slider; import org.apache.slider.api.RoleKeys; -import org.apache.slider.api.resource.Application; -import org.apache.slider.api.resource.Component; import org.apache.slider.api.types.ContainerInformation; import org.apache.slider.common.SliderKeys; import org.apache.slider.common.SliderXmlConfKeys; @@ -2540,14 +2538,4 @@ public static long getTimeRange(org.apache.slider.api.resource long totalMinutes = days * 24 * 60 + hours * 24 + minutes; return totalMinutes * 60 + seconds; } - - public static void resolve(Application application) { - org.apache.slider.api.resource.Configuration global = application - .getConfiguration(); - for (Component component : application.getComponents()) { - mergeMapsIgnoreDuplicateKeys(component.getConfiguration().getProperties(), - global.getProperties()); - } - // TODO merge other information to components - } -} \ No newline at end of file +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java index 84dde08..7e198e8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java @@ -107,7 +107,6 @@ import org.apache.slider.core.main.LauncherExitCodes; import org.apache.slider.core.main.RunService; import org.apache.slider.core.main.ServiceLauncher; -import org.apache.slider.core.persist.JsonSerDeser; import org.apache.slider.core.registry.info.CustomRegistryConstants; import org.apache.slider.providers.ProviderCompleted; import org.apache.slider.providers.ProviderService; @@ -157,7 +156,7 @@ import org.apache.slider.server.services.workflow.WorkflowExecutorService; import org.apache.slider.server.services.workflow.WorkflowRpcService; import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders; -import org.codehaus.jackson.map.PropertyNamingStrategy; +import org.apache.slider.util.ServiceApiUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -389,9 +388,6 @@ */ private boolean securityEnabled; private ContentCache contentCache; - private static final JsonSerDeser jsonSerDeser = - new JsonSerDeser(Application.class, - PropertyNamingStrategy.CAMEL_CASE_TO_LOWER_CASE_WITH_UNDERSCORES); /** * resource limits @@ -590,9 +586,7 @@ private int createAndRunCluster(String appName) throws Throwable { Path appDir = new Path((serviceArgs.getAppDefDir())); SliderFileSystem fs = getClusterFS(); fs.setAppDir(appDir); - Path appJson = new Path(appDir, appName + ".json"); - log.info("Loading application definition from " + appJson); - application = jsonSerDeser.load(fs.getFileSystem(), appJson); + application = ServiceApiUtil.loadApplication(fs, appName); log.info("Application Json: " + application); stateForProviders.setApplicationName(appName); Configuration serviceConf = getConfig(); @@ -874,11 +868,6 @@ private int createAndRunCluster(String appName) throws Throwable { scheduleFailureWindowResets(application.getConfiguration()); scheduleEscalation(application.getConfiguration()); - for (Component component : application.getComponents()) { - // Merge app-level configuration into component level configuration - component.getConfiguration().mergeFrom(application.getConfiguration()); - } - try { // schedule YARN Registry registration queue(new ActionRegisterServiceInstance(appName, appid, application)); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java index ac89ed8..f72afd8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java @@ -21,7 +21,7 @@ String ERROR_APPLICATION_NAME_INVALID = "Application name is either empty or not provided"; String ERROR_APPLICATION_NAME_INVALID_FORMAT = - "Application name is not valid - only lower case letters, digits," + "Application name %s is not valid - only lower case letters, digits," + " underscore and hyphen are allowed"; String ERROR_APPLICATION_NOT_RUNNING = "Application not running"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/ServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/ServiceApiUtil.java index d7c72a3..d2db4db 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/ServiceApiUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/ServiceApiUtil.java @@ -28,12 +28,17 @@ import org.apache.slider.api.resource.ConfigFile; import org.apache.slider.api.resource.Configuration; import org.apache.slider.api.resource.Resource; +import org.apache.slider.common.tools.SliderFileSystem; import org.apache.slider.common.tools.SliderUtils; +import org.apache.slider.core.persist.JsonSerDeser; +import org.apache.slider.providers.AbstractClientProvider; +import org.apache.slider.providers.SliderProviderFactory; +import org.codehaus.jackson.map.PropertyNamingStrategy; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; -import java.nio.file.Paths; +import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -41,102 +46,138 @@ public class ServiceApiUtil { private static final Logger log = LoggerFactory.getLogger(ServiceApiUtil.class); + private static final JsonSerDeser JSON_SER_DESER = + new JsonSerDeser<>(Application.class, + PropertyNamingStrategy.CAMEL_CASE_TO_LOWER_CASE_WITH_UNDERSCORES); + @VisibleForTesting public static void validateApplicationPayload(Application application, - FileSystem fs) throws IOException { + SliderFileSystem fs) throws IOException { if (StringUtils.isEmpty(application.getName())) { throw new IllegalArgumentException( RestApiErrorMessages.ERROR_APPLICATION_NAME_INVALID); } if (!SliderUtils.isClusternameValid(application.getName())) { - throw new IllegalArgumentException( - RestApiErrorMessages.ERROR_APPLICATION_NAME_INVALID_FORMAT); + throw new IllegalArgumentException(String.format( + RestApiErrorMessages.ERROR_APPLICATION_NAME_INVALID_FORMAT, + application.getName())); } // If the application has no components do top-level checks if (!hasComponent(application)) { - // artifact - if (application.getArtifact() == null) { - throw new IllegalArgumentException( - RestApiErrorMessages.ERROR_ARTIFACT_INVALID); - } - if (StringUtils.isEmpty(application.getArtifact().getId())) { - throw new IllegalArgumentException( - RestApiErrorMessages.ERROR_ARTIFACT_ID_INVALID); - } + // If artifact is of type APPLICATION, read other application components + if (application.getArtifact() != null && application.getArtifact() + .getType() == Artifact.TypeEnum.APPLICATION) { + if (StringUtils.isEmpty(application.getArtifact().getId())) { + throw new IllegalArgumentException( + RestApiErrorMessages.ERROR_ARTIFACT_ID_INVALID); + } + Application otherApplication = loadApplication(fs, + application.getArtifact().getId()); + application.setComponents(otherApplication.getComponents()); + application.setArtifact(null); + SliderUtils.mergeMapsIgnoreDuplicateKeys(application.getQuicklinks(), + otherApplication.getQuicklinks()); + } else { + // artifact + AbstractClientProvider clientProvider = SliderProviderFactory + .getClientProvider(application.getArtifact()); + clientProvider.validateArtifact(application.getArtifact(), fs + .getFileSystem()); - // If artifact is of type APPLICATION, add a slider specific property - if (application.getArtifact().getType() - == Artifact.TypeEnum.APPLICATION) { - if (application.getConfiguration() == null) { - application.setConfiguration(new Configuration()); + // resource + validateApplicationResource(application.getResource(), null); + + // container size + if (application.getNumberOfContainers() == null + || application.getNumberOfContainers() < 0) { + throw new IllegalArgumentException( + RestApiErrorMessages.ERROR_CONTAINERS_COUNT_INVALID + ": " + + application.getNumberOfContainers()); } + validateConfigFile(application.getConfiguration().getFiles(), fs + .getFileSystem()); + clientProvider.validateConfigFiles(application.getConfiguration() + .getFiles(), fs.getFileSystem()); + // Since it is a simple app with no components, create a default + // component + application.getComponents().add(createDefaultComponent(application)); } - // resource - validateApplicationResource(application.getResource(), null, - application.getArtifact().getType()); - - // container size - if (application.getNumberOfContainers() == null - || application.getNumberOfContainers() < 0) { - throw new IllegalArgumentException( - RestApiErrorMessages.ERROR_CONTAINERS_COUNT_INVALID + ": " - + application.getNumberOfContainers()); + } + // Now run checks for each component. + // Let global values take effect if component level values are not + // provided. + Artifact globalArtifact = application.getArtifact(); + Resource globalResource = application.getResource(); + Configuration globalConf = application.getConfiguration(); + Long globalNumberOfContainers = application.getNumberOfContainers(); + Set componentNames = new HashSet<>(); + List componentsToRemove = new ArrayList<>(); + List componentsToAdd = new ArrayList<>(); + for (Component comp : application.getComponents()) { + if (componentNames.contains(comp.getName())) { + throw new IllegalArgumentException("Component name collision: " + + comp.getName()); } - validateConfigFile(application.getConfiguration().getFiles(), fs); - // Since it is a simple app with no components, create a default component - application.getComponents().add(createDefaultComponent(application)); - } else { - // If the application has components, then run checks for each component. - // Let global values take effect if component level values are not - // provided. - Artifact globalArtifact = application.getArtifact(); - Resource globalResource = application.getResource(); - Long globalNumberOfContainers = application.getNumberOfContainers(); - for (Component comp : application.getComponents()) { - // artifact - if (comp.getArtifact() == null) { - comp.setArtifact(globalArtifact); - } - // If still null raise validation exception - if (comp.getArtifact() == null) { - throw new IllegalArgumentException(String - .format(RestApiErrorMessages.ERROR_ARTIFACT_FOR_COMP_INVALID, - comp.getName())); - } + // artifact + if (comp.getArtifact() == null) { + comp.setArtifact(globalArtifact); + } + // configuration + comp.getConfiguration().mergeFrom(globalConf); + // If artifact is of type APPLICATION, read other application components + if (comp.getArtifact() != null && comp.getArtifact().getType() == + Artifact.TypeEnum.APPLICATION) { if (StringUtils.isEmpty(comp.getArtifact().getId())) { - throw new IllegalArgumentException(String - .format(RestApiErrorMessages.ERROR_ARTIFACT_ID_FOR_COMP_INVALID, - comp.getName())); + throw new IllegalArgumentException( + RestApiErrorMessages.ERROR_ARTIFACT_ID_INVALID); } - - // If artifact is of type APPLICATION, add a slider specific property - if (comp.getArtifact().getType() == Artifact.TypeEnum.APPLICATION) { - if (comp.getConfiguration() == null) { - comp.setConfiguration(new Configuration()); + componentsToRemove.add(comp); + List applicationComponents = getApplicationComponents(fs, + comp.getArtifact().getId()); + for (Component c : applicationComponents) { + if (componentNames.contains(c.getName())) { + // TODO allow name collisions? see AppState#roles + // TODO or add prefix to external component names? + throw new IllegalArgumentException("Component name collision: " + + c.getName()); } - comp.setName(comp.getArtifact().getId()); + componentNames.add(c.getName()); } + componentsToAdd.addAll(applicationComponents); + } else { + componentNames.add(comp.getName()); + } + } + application.getComponents().removeAll(componentsToRemove); + application.getComponents().addAll(componentsToAdd); - // resource - if (comp.getResource() == null) { - comp.setResource(globalResource); - } - validateApplicationResource(comp.getResource(), comp, - comp.getArtifact().getType()); + for (Component comp : application.getComponents()) { + AbstractClientProvider compClientProvider = SliderProviderFactory + .getClientProvider(comp.getArtifact()); + compClientProvider.validateArtifact(comp.getArtifact(), fs + .getFileSystem()); - // container count - if (comp.getNumberOfContainers() == null) { - comp.setNumberOfContainers(globalNumberOfContainers); - } - if (comp.getNumberOfContainers() == null - || comp.getNumberOfContainers() < 0) { - throw new IllegalArgumentException(String.format( - RestApiErrorMessages.ERROR_CONTAINERS_COUNT_FOR_COMP_INVALID - + ": " + comp.getNumberOfContainers(), comp.getName())); - } - validateConfigFile(comp.getConfiguration().getFiles(), fs); + // resource + if (comp.getResource() == null) { + comp.setResource(globalResource); + } + validateApplicationResource(comp.getResource(), comp); + + // container count + if (comp.getNumberOfContainers() == null) { + comp.setNumberOfContainers(globalNumberOfContainers); + } + if (comp.getNumberOfContainers() == null + || comp.getNumberOfContainers() < 0) { + throw new IllegalArgumentException(String.format( + RestApiErrorMessages.ERROR_CONTAINERS_COUNT_FOR_COMP_INVALID + + ": " + comp.getNumberOfContainers(), comp.getName())); } + validateConfigFile(comp.getConfiguration().getFiles(), fs + .getFileSystem()); + compClientProvider.validateConfigFiles(comp.getConfiguration() + .getFiles(), fs.getFileSystem()); } // Application lifetime if not specified, is set to unlimited lifetime @@ -145,13 +186,38 @@ public static void validateApplicationPayload(Application application, } } + @VisibleForTesting + public static List getApplicationComponents(SliderFileSystem + fs, String appName) throws IOException { + return loadApplication(fs, appName).getComponents(); + } + + public static Application loadApplication(SliderFileSystem fs, String + appName) throws IOException { + Path appJson = getAppJsonPath(fs, appName); + log.info("Loading application definition from " + appJson); + Application externalApplication = JSON_SER_DESER.load(fs + .getFileSystem(), appJson); + return externalApplication; + } + + public static Path getAppJsonPath(SliderFileSystem fs, String appName) { + Path appDir = fs.buildClusterDirPath(appName); + Path appJson = new Path(appDir, appName + ".json"); + return appJson; + } + // 1) Verify the src_file exists and non-empty for template - // 2) dest_file is absolute path - private static void validateConfigFile(List list, FileSystem fs) + @VisibleForTesting + public static void validateConfigFile(List list, FileSystem fs) throws IOException { Set destFileSet = new HashSet<>(); for (ConfigFile file : list) { + if (file.getType() == null) { + throw new IllegalArgumentException("File type is empty"); + } + if (file.getType().equals(ConfigFile.TypeEnum.TEMPLATE) && StringUtils .isEmpty(file.getSrcFile())) { throw new IllegalArgumentException( @@ -170,11 +236,6 @@ private static void validateConfigFile(List list, FileSystem fs) if (StringUtils.isEmpty(file.getDestFile())) { throw new IllegalArgumentException("Dest_file is empty."); } - // validate dest_file is absolute - if (!Paths.get(file.getDestFile()).isAbsolute()) { - throw new IllegalArgumentException( - "Dest_file must be absolute path: " + file.getDestFile()); - } if (destFileSet.contains(file.getDestFile())) { throw new IllegalArgumentException( @@ -186,11 +247,8 @@ private static void validateConfigFile(List list, FileSystem fs) private static void validateApplicationResource(Resource resource, - Component comp, Artifact.TypeEnum artifactType) { + Component comp) { // Only apps/components of type APPLICATION can skip resource requirement - if (resource == null && artifactType == Artifact.TypeEnum.APPLICATION) { - return; - } if (resource == null) { throw new IllegalArgumentException( comp == null ? RestApiErrorMessages.ERROR_RESOURCE_INVALID : String diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestKeytabCommandOptions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestKeytabCommandOptions.java index 07d8c10..59ccda7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestKeytabCommandOptions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestKeytabCommandOptions.java @@ -34,6 +34,7 @@ import org.apache.slider.core.exceptions.SliderException; import org.apache.slider.core.main.ServiceLauncher; import org.apache.slider.utils.SliderTestBase; +import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -50,6 +51,7 @@ public class TestKeytabCommandOptions extends SliderTestBase { private static SliderFileSystem testFileSystem; + private File testFolderDir; @Before public void setupFilesystem() throws IOException { @@ -57,11 +59,18 @@ public void setupFilesystem() throws IOException { YarnConfiguration configuration = SliderUtils.createConfiguration(); fileSystem.setConf(configuration); testFileSystem = new SliderFileSystem(fileSystem, configuration); - File testFolderDir = new File(testFileSystem + testFolderDir = new File(testFileSystem .buildKeytabInstallationDirPath("").toUri().getPath()); FileUtils.deleteDirectory(testFolderDir); } + @After + public void cleanup() throws IOException { + if (testFolderDir != null && testFolderDir.exists()) { + FileUtils.deleteDirectory(testFolderDir); + } + } + @Test public void testInstallKeytab() throws Throwable { // create a mock keytab file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestMiscSliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestMiscSliderUtils.java index bf6ee2c..3392ff8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestMiscSliderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestMiscSliderUtils.java @@ -45,5 +45,7 @@ public void testPurgeTempDir() throws Throwable { assertTrue(fs.exists(inst)); sliderFileSystem.purgeAppInstanceTempFiles(CLUSTER1); assertFalse(fs.exists(inst)); + + fs.delete(sliderFileSystem.buildClusterDirPath(CLUSTER1), true); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/ExampleConfResources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/ExampleConfResources.java index f13fbcc..13beb9b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/ExampleConfResources.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/ExampleConfResources.java @@ -32,15 +32,17 @@ public final class ExampleConfResources { public static final String APP_JSON = "app.json"; - public static final String APP_RES = "app-resolved.json"; public static final String OVERRIDE_JSON = "app-override.json"; - public static final String OVERRIDE_RES = "app-override-resolved.json"; + public static final String DEFAULT_JSON = "default.json"; + public static final String EXTERNAL_JSON_0 = "external0.json"; + public static final String EXTERNAL_JSON_1 = "external1.json"; + public static final String EXTERNAL_JSON_2 = "external2.json"; public static final String PACKAGE = "/org/apache/slider/core/conf/examples/"; - private static final String[] ALL_EXAMPLES = {APP_JSON, APP_RES, - OVERRIDE_JSON, OVERRIDE_RES}; + private static final String[] ALL_EXAMPLES = {APP_JSON, OVERRIDE_JSON, + DEFAULT_JSON}; public static final List ALL_EXAMPLE_RESOURCES = new ArrayList<>(); static { @@ -55,4 +57,8 @@ private ExampleConfResources() { static Application loadResource(String name) throws IOException { return JSON_SER_DESER.fromResource(PACKAGE + name); } + + public static String resourceName(String name) { + return "target/test-classes" + PACKAGE + name; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestConfTreeLoadExamples.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestConfTreeLoadExamples.java index 48b0736..bafbdfc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestConfTreeLoadExamples.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestConfTreeLoadExamples.java @@ -18,8 +18,11 @@ package org.apache.slider.core.conf; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.slider.api.resource.Application; -import org.apache.slider.common.tools.SliderUtils; +import org.apache.slider.common.tools.SliderFileSystem; +import org.apache.slider.util.ServiceApiUtil; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -29,6 +32,10 @@ import java.util.Collection; import static org.apache.slider.utils.SliderTestUtils.JSON_SER_DESER; +import static org.easymock.EasyMock.anyObject; +import static org.easymock.EasyMock.createNiceMock; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.replay; /** * Test loading example resources. @@ -56,7 +63,15 @@ public TestConfTreeLoadExamples(String resource) { public void testLoadResource() throws Throwable { try { Application application = JSON_SER_DESER.fromResource(resource); - SliderUtils.resolve(application); + + SliderFileSystem sfs = createNiceMock(SliderFileSystem.class); + FileSystem mockFs = createNiceMock(FileSystem.class); + expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes(); + expect(sfs.buildClusterDirPath(anyObject())).andReturn( + new Path("cluster_dir_path")).anyTimes(); + replay(sfs, mockFs); + + ServiceApiUtil.validateApplicationPayload(application, sfs); } catch (Exception e) { throw new Exception("exception loading " + resource + ":" + e.toString()); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestConfigurationResolve.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestConfigurationResolve.java index 285ddfa..ac2cc2e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestConfigurationResolve.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestConfigurationResolve.java @@ -18,20 +18,36 @@ package org.apache.slider.core.conf; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.slider.api.resource.Application; +import org.apache.slider.api.resource.ConfigFile; +import org.apache.slider.api.resource.ConfigFile.TypeEnum; import org.apache.slider.api.resource.Configuration; +import org.apache.slider.common.tools.SliderFileSystem; import org.apache.slider.common.tools.SliderUtils; +import org.apache.slider.util.ServiceApiUtil; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + import static org.apache.slider.api.InternalKeys.CHAOS_MONKEY_INTERVAL; import static org.apache.slider.api.InternalKeys.DEFAULT_CHAOS_MONKEY_INTERVAL_DAYS; import static org.apache.slider.api.InternalKeys.DEFAULT_CHAOS_MONKEY_INTERVAL_HOURS; import static org.apache.slider.api.InternalKeys.DEFAULT_CHAOS_MONKEY_INTERVAL_MINUTES; import static org.apache.slider.core.conf.ExampleConfResources.APP_JSON; import static org.apache.slider.core.conf.ExampleConfResources.OVERRIDE_JSON; +import static org.easymock.EasyMock.anyObject; +import static org.easymock.EasyMock.createNiceMock; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.replay; /** * Test global configuration resolution. @@ -48,17 +64,21 @@ public void testOverride() throws Throwable { Configuration global = orig.getConfiguration(); assertEquals("a", global.getProperty("g1")); assertEquals("b", global.getProperty("g2")); + assertEquals(2, global.getFiles().size()); Configuration simple = orig.getComponent("simple").getConfiguration(); assertEquals(0, simple.getProperties().size()); + assertEquals(1, simple.getFiles().size()); Configuration master = orig.getComponent("master").getConfiguration(); assertEquals("m", master.getProperty("name")); assertEquals("overridden", master.getProperty("g1")); + assertEquals(0, master.getFiles().size()); Configuration worker = orig.getComponent("worker").getConfiguration(); LOG.info("worker = {}", worker); assertEquals(3, worker.getProperties().size()); + assertEquals(0, worker.getFiles().size()); assertEquals("worker", worker.getProperty("name")); assertEquals("overridden-by-worker", worker.getProperty("g1")); @@ -66,18 +86,36 @@ public void testOverride() throws Throwable { assertEquals("1000", worker.getProperty("timeout")); // here is the resolution - SliderUtils.resolve(orig); + SliderFileSystem sfs = createNiceMock(SliderFileSystem.class); + FileSystem mockFs = createNiceMock(FileSystem.class); + expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes(); + expect(sfs.buildClusterDirPath(anyObject())).andReturn( + new Path("cluster_dir_path")).anyTimes(); + replay(sfs, mockFs); + ServiceApiUtil.validateApplicationPayload(orig, sfs); global = orig.getConfiguration(); LOG.info("global = {}", global); assertEquals("a", global.getProperty("g1")); assertEquals("b", global.getProperty("g2")); + assertEquals(2, global.getFiles().size()); simple = orig.getComponent("simple").getConfiguration(); assertEquals(2, simple.getProperties().size()); assertEquals("a", simple.getProperty("g1")); assertEquals("b", simple.getProperty("g2")); - + assertEquals(2, simple.getFiles().size()); + + Set files = new HashSet<>(); + Map props = new HashMap<>(); + props.put("k1", "overridden"); + props.put("k2", "v2"); + files.add(new ConfigFile().destFile("file1").type(TypeEnum + .PROPERTIES).props(props)); + files.add(new ConfigFile().destFile("file2").type(TypeEnum + .XML).props(Collections.singletonMap("k3", "v3"))); + assertTrue(files.contains(simple.getFiles().get(0))); + assertTrue(files.contains(simple.getFiles().get(1))); master = orig.getComponent("master").getConfiguration(); LOG.info("master = {}", master); @@ -85,6 +123,17 @@ public void testOverride() throws Throwable { assertEquals("m", master.getProperty("name")); assertEquals("overridden", master.getProperty("g1")); assertEquals("b", master.getProperty("g2")); + assertEquals(2, master.getFiles().size()); + + props.put("k1", "v1"); + files.clear(); + files.add(new ConfigFile().destFile("file1").type(TypeEnum + .PROPERTIES).props(props)); + files.add(new ConfigFile().destFile("file2").type(TypeEnum + .XML).props(Collections.singletonMap("k3", "v3"))); + + assertTrue(files.contains(master.getFiles().get(0))); + assertTrue(files.contains(master.getFiles().get(1))); worker = orig.getComponent("worker").getConfiguration(); LOG.info("worker = {}", worker); @@ -94,12 +143,14 @@ public void testOverride() throws Throwable { assertEquals("overridden-by-worker", worker.getProperty("g1")); assertEquals("b", worker.getProperty("g2")); assertEquals("1000", worker.getProperty("timeout")); + assertEquals(2, worker.getFiles().size()); + assertTrue(files.contains(worker.getFiles().get(0))); + assertTrue(files.contains(worker.getFiles().get(1))); } @Test public void testTimeIntervalLoading() throws Throwable { - Application orig = ExampleConfResources.loadResource(APP_JSON); Configuration conf = orig.getConfiguration(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestBuildApplicationComponent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestBuildApplicationComponent.java new file mode 100644 index 0000000..2b2be56 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestBuildApplicationComponent.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.providers; + +import org.apache.slider.api.resource.Component; +import org.apache.slider.client.SliderClient; +import org.apache.slider.common.params.SliderActions; +import org.apache.slider.common.tools.SliderFileSystem; +import org.apache.slider.core.conf.ExampleConfResources; +import org.apache.slider.core.main.ServiceLauncher; +import org.apache.slider.util.ServiceApiUtil; +import org.apache.slider.utils.YarnZKMiniClusterTestBase; +import org.junit.Test; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static org.apache.slider.common.params.Arguments.ARG_APPDEF; + +/** + * Test for building / resolving components of type APPLICATION. + */ +public class TestBuildApplicationComponent extends YarnZKMiniClusterTestBase { + + private static void checkComponentNames(List components, + Set names) { + assertEquals(names.size(), components.size()); + for (Component comp : components) { + assertTrue(names.contains(comp.getName())); + } + } + + public void buildAndCheckComponents(String appName, String appDef, + SliderFileSystem sfs, Set names) throws Throwable { + ServiceLauncher launcher = createOrBuildCluster( + SliderActions.ACTION_BUILD, appName, Arrays.asList(ARG_APPDEF, + ExampleConfResources.resourceName(appDef)), true, false); + SliderClient sliderClient = launcher.getService(); + addToTeardown(sliderClient); + + // verify the cluster exists + assertEquals(0, sliderClient.actionExists(appName, false)); + // verify generated conf + List components = ServiceApiUtil.getApplicationComponents(sfs, + appName); + checkComponentNames(components, names); + } + + @Test + public void testExternalComponentBuild() throws Throwable { + String clustername = createMiniCluster("", getConfiguration(), 1, true); + + describe("verify external components"); + + SliderFileSystem sfs = createSliderFileSystem(); + + Set nameSet = new HashSet<>(); + nameSet.add("simple"); + nameSet.add("master"); + nameSet.add("worker"); + + buildAndCheckComponents("app-1", ExampleConfResources.APP_JSON, sfs, + nameSet); + buildAndCheckComponents("external-0", ExampleConfResources + .EXTERNAL_JSON_0, sfs, nameSet); + + nameSet.add("other"); + + buildAndCheckComponents("external-1", ExampleConfResources + .EXTERNAL_JSON_1, sfs, nameSet); + + nameSet.add("another"); + + buildAndCheckComponents("external-2", ExampleConfResources + .EXTERNAL_JSON_2, sfs, nameSet); + + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestDefaultProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestDefaultProvider.java new file mode 100644 index 0000000..1f694ff --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestDefaultProvider.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.providers; + +import org.apache.slider.api.resource.Application; +import org.apache.slider.client.SliderClient; +import org.apache.slider.common.params.SliderActions; +import org.apache.slider.core.conf.ExampleConfResources; +import org.apache.slider.core.main.ServiceLauncher; +import org.apache.slider.utils.YarnZKMiniClusterTestBase; +import org.junit.Ignore; +import org.junit.Test; + +import java.util.Arrays; + +import static org.apache.slider.common.params.Arguments.ARG_APPDEF; + +/** + * Simple end-to-end test. + */ +public class TestDefaultProvider extends YarnZKMiniClusterTestBase { + + // TODO figure out how to run client commands against minicluster + // (currently errors out unable to find containing jar of AM for upload) + @Ignore + @Test + public void testDefaultProvider() throws Throwable { + createMiniCluster("", getConfiguration(), 1, true); + String appName = "default-1"; + + describe("verify default provider"); + + String appDef = ExampleConfResources.resourceName(ExampleConfResources + .DEFAULT_JSON); + + ServiceLauncher launcher = createOrBuildCluster( + SliderActions.ACTION_CREATE, appName, Arrays.asList(ARG_APPDEF, + appDef), true, true); + SliderClient sliderClient = launcher.getService(); + addToTeardown(sliderClient); + + Application application = sliderClient.actionStatus(appName); + assertEquals(1L, application.getContainers().size()); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/BaseMockAppStateAATest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/BaseMockAppStateAATest.java index c1f2886..6f4ca42 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/BaseMockAppStateAATest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/BaseMockAppStateAATest.java @@ -43,7 +43,7 @@ @Override public Application buildApplication() { Application application = factory.newApplication(0, 0, 0) - .name(getTestName()); + .name(getValidTestName()); application.getComponent(ROLE1).getConfiguration().setProperty( COMPONENT_PLACEMENT_POLICY, Integer.toString(PlacementPolicy .ANTI_AFFINITY_REQUIRED)); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.java index eb25b40..571e9d9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.java @@ -362,7 +362,7 @@ public void testAMRestart() throws Throwable { // now destroy the app state AppStateBindingInfo bindingInfo = buildBindingInfo(); bindingInfo.application = factory.newApplication(0, 0, desiredAA).name( - getTestName()); + getValidTestName()); bindingInfo.application.getComponent(ROLE2) .getConfiguration().setProperty(COMPONENT_PLACEMENT_POLICY, Integer.toString(PlacementPolicy.ANTI_AFFINITY_REQUIRED)); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.java index ea0dcf4..9cbda4f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.java @@ -203,7 +203,7 @@ public void testRecurrentStartupFailureWithUnlimitedFailures() throws // Update instance definition to allow containers to fail any number of // times AppStateBindingInfo bindingInfo = buildBindingInfo(); - bindingInfo.application.getConfiguration().setProperty( + bindingInfo.application.getComponent(ROLE0).getConfiguration().setProperty( ResourceKeys.CONTAINER_FAILURE_THRESHOLD, "0"); appState = new MockAppState(bindingInfo); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.java index 6d8e963..7f7f93a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.java @@ -38,6 +38,7 @@ import org.slf4j.LoggerFactory; import java.io.File; +import java.io.IOException; import java.util.Collections; /** @@ -65,7 +66,7 @@ public MockYarnEngine createYarnEngine() { } @Override - public AppStateBindingInfo buildBindingInfo() { + public AppStateBindingInfo buildBindingInfo() throws IOException { AppStateBindingInfo bindingInfo = super.buildBindingInfo(); bindingInfo.releaseSelector = new MostRecentContainerReleaseSelector(); return bindingInfo; @@ -145,7 +146,7 @@ public void testHistoryFlexSaveResetLoad() throws Throwable { appState = new MockAppState(); AppStateBindingInfo binding2 = buildBindingInfo(); binding2.application = factory.newApplication(0, 0, 0) - .name(getTestName()); + .name(getValidTestName()); binding2.historyPath = historyPath2; appState.buildInstance(binding2); // on this read there won't be the right number of roles diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.java index b0634bf..d9c675d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.java @@ -70,7 +70,7 @@ public void testRebuild() throws Throwable { AppStateBindingInfo bindingInfo = buildBindingInfo(); bindingInfo.application = factory.newApplication(r0, r1, r2) - .name(getTestName()); + .name(getValidTestName()); bindingInfo.liveContainers = containers; appState = new MockAppState(bindingInfo); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java index b7e967f..703d65f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java @@ -30,6 +30,7 @@ import org.apache.slider.server.appmaster.state.RoleStatus; import org.junit.Test; +import java.io.IOException; import java.util.Collections; import java.util.List; import java.util.Map; @@ -59,7 +60,7 @@ public MockYarnEngine createYarnEngine() { } @Override - public AppStateBindingInfo buildBindingInfo() { + public AppStateBindingInfo buildBindingInfo() throws IOException { AppStateBindingInfo bindingInfo = super.buildBindingInfo(); bindingInfo.releaseSelector = new MostRecentContainerReleaseSelector(); return bindingInfo; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.java index d382c8a..4aa5895 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.java @@ -40,7 +40,7 @@ @Override public Application buildApplication() { - return factory.newApplication(1, 0, 0).name(getTestName()); + return factory.newApplication(1, 0, 0).name(getValidTestName()); } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java index 69abccf..6a2bd2d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java @@ -51,6 +51,7 @@ import org.apache.slider.server.appmaster.state.RoleInstance; import org.apache.slider.server.appmaster.state.RoleStatus; import org.apache.slider.server.appmaster.state.StateAccessForProviders; +import org.apache.slider.util.ServiceApiUtil; import org.apache.slider.utils.SliderTestBase; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -62,6 +63,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Locale; import java.util.Map.Entry; /** @@ -118,7 +120,7 @@ protected void initApp() historyPath = new Path(historyWorkDir.toURI()); fs.delete(historyPath, true); appState = new MockAppState(buildBindingInfo()); - stateAccess = new ProviderAppState(getTestName(), appState); + stateAccess = new ProviderAppState(getValidTestName(), appState); } /** @@ -127,9 +129,11 @@ protected void initApp() * from {@link #buildApplication()} ()} * @return */ - protected AppStateBindingInfo buildBindingInfo() { + protected AppStateBindingInfo buildBindingInfo() throws IOException { AppStateBindingInfo binding = new AppStateBindingInfo(); binding.application = buildApplication(); + ServiceApiUtil.validateApplicationPayload(binding.application, + sliderFileSystem); //binding.roles = new ArrayList<>(factory.ROLES); binding.fs = fs; binding.historyPath = historyPath; @@ -142,7 +146,7 @@ protected AppStateBindingInfo buildBindingInfo() { * @return the instance definition */ public Application buildApplication() { - return factory.newApplication(0, 0, 0).name(getTestName()); + return factory.newApplication(0, 0, 0).name(getValidTestName()); } /** @@ -153,6 +157,10 @@ public String getTestName() { return methodName.getMethodName(); } + public String getValidTestName() { + return getTestName().toLowerCase(Locale.ENGLISH); + } + public RoleStatus getRole0Status() { return lookupRole(ROLE0); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockFactory.java index 2ac5087..d27cd33 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockFactory.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockFactory.java @@ -32,6 +32,7 @@ import org.apache.slider.api.ResourceKeys; import org.apache.slider.api.resource.Application; import org.apache.slider.api.resource.Component; +import org.apache.slider.api.resource.Resource; import org.apache.slider.providers.PlacementPolicy; import org.apache.slider.providers.ProviderRole; @@ -190,6 +191,7 @@ public MockContainer newContainer(AMRMClient.ContainerRequest req, String */ public Application newApplication(long r1, long r2, long r3) { Application application = new Application(); + application.setResource(new Resource().memory("256")); application.getConfiguration().setProperty(ResourceKeys .NODE_FAILURE_THRESHOLD, Integer.toString(NODE_FAILURE_THRESHOLD)); List components = application.getComponents(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestServiceApiUtil.java new file mode 100644 index 0000000..e35c485 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestServiceApiUtil.java @@ -0,0 +1,287 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.utils; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.slider.api.resource.Application; +import org.apache.slider.api.resource.Artifact; +import org.apache.slider.api.resource.ConfigFile; +import org.apache.slider.api.resource.ConfigFile.TypeEnum; +import org.apache.slider.api.resource.Resource; +import org.apache.slider.common.tools.SliderFileSystem; +import org.apache.slider.util.ServiceApiUtil; +import org.junit.Assert; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.slider.util.RestApiConstants.DEFAULT_COMPONENT_NAME; +import static org.apache.slider.util.RestApiConstants.DEFAULT_UNLIMITED_LIFETIME; +import static org.apache.slider.util.RestApiErrorMessages.*; +import static org.apache.slider.util.RestApiErrorMessages.ERROR_CONTAINERS_COUNT_INVALID; +import static org.apache.slider.util.RestApiErrorMessages.ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_NOT_SUPPORTED; +import static org.apache.slider.util.RestApiErrorMessages.ERROR_RESOURCE_PROFILE_NOT_SUPPORTED_YET; +import static org.easymock.EasyMock.anyObject; +import static org.easymock.EasyMock.createNiceMock; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.replay; + +/** + * Test for ServiceApiUtil helper methods. + */ +public class TestServiceApiUtil { + private static final Logger LOG = LoggerFactory + .getLogger(TestServiceApiUtil.class); + private static final String EXCEPTION_PREFIX = "Should have thrown " + + "exception: "; + private static final String NO_EXCEPTION_PREFIX = "Should not have thrown " + + "exception: "; + + @Test(timeout = 90000) + public void testValidateApplicationPostPayload() throws Exception { + SliderFileSystem sfs = createNiceMock(SliderFileSystem.class); + FileSystem mockFs = createNiceMock(FileSystem.class); + expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes(); + expect(sfs.buildClusterDirPath(anyObject())).andReturn( + new Path("cluster_dir_path")).anyTimes(); + replay(sfs, mockFs); + + Application app = new Application(); + + // no name + try { + ServiceApiUtil.validateApplicationPayload(app, sfs); + Assert.fail(EXCEPTION_PREFIX + "application with no name"); + } catch (IllegalArgumentException e) { + Assert.assertEquals(ERROR_APPLICATION_NAME_INVALID, e.getMessage()); + } + + // bad format name + String[] badNames = {"4finance", "Finance", "finance@home"}; + for (String badName : badNames) { + app.setName(badName); + try { + ServiceApiUtil.validateApplicationPayload(app, sfs); + Assert.fail(EXCEPTION_PREFIX + "application with bad name " + badName); + } catch (IllegalArgumentException e) { + Assert.assertEquals(String.format( + ERROR_APPLICATION_NAME_INVALID_FORMAT, badName), e.getMessage()); + } + } + + // resource not specified + app.setName("finance_home"); + try { + ServiceApiUtil.validateApplicationPayload(app, sfs); + Assert.fail(EXCEPTION_PREFIX + "application with no resource"); + } catch (IllegalArgumentException e) { + Assert.assertEquals(ERROR_RESOURCE_INVALID, e.getMessage()); + } + + // memory not specified + Resource res = new Resource(); + app.setResource(res); + try { + ServiceApiUtil.validateApplicationPayload(app, sfs); + Assert.fail(EXCEPTION_PREFIX + "application with no memory"); + } catch (IllegalArgumentException e) { + Assert.assertEquals(ERROR_RESOURCE_MEMORY_INVALID, e.getMessage()); + } + + // invalid no of cpus + res.setMemory("100mb"); + res.setCpus(-2); + try { + ServiceApiUtil.validateApplicationPayload(app, sfs); + Assert.fail( + EXCEPTION_PREFIX + "application with invalid no of cpups"); + } catch (IllegalArgumentException e) { + Assert.assertEquals(ERROR_RESOURCE_CPUS_INVALID_RANGE, e.getMessage()); + } + + // number of containers not specified + res.setCpus(2); + try { + ServiceApiUtil.validateApplicationPayload(app, sfs); + Assert.fail(EXCEPTION_PREFIX + "application with no container count"); + } catch (IllegalArgumentException e) { + Assert.assertTrue(e.getMessage() + .contains(ERROR_CONTAINERS_COUNT_INVALID)); + } + + // specifying profile along with cpus/memory raises exception + res.setProfile("hbase_finance_large"); + try { + ServiceApiUtil.validateApplicationPayload(app, sfs); + Assert.fail(EXCEPTION_PREFIX + + "application with resource profile along with cpus/memory"); + } catch (IllegalArgumentException e) { + Assert.assertEquals(ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_NOT_SUPPORTED, + e.getMessage()); + } + + // currently resource profile alone is not supported. + // TODO: remove the next test once resource profile alone is supported. + res.setCpus(null); + res.setMemory(null); + try { + ServiceApiUtil.validateApplicationPayload(app, sfs); + Assert.fail(EXCEPTION_PREFIX + "application with resource profile only"); + } catch (IllegalArgumentException e) { + Assert.assertEquals(ERROR_RESOURCE_PROFILE_NOT_SUPPORTED_YET, + e.getMessage()); + } + + // unset profile here and add cpus/memory back + res.setProfile(null); + res.setCpus(2); + res.setMemory("2gb"); + + // null number of containers + try { + ServiceApiUtil.validateApplicationPayload(app, sfs); + Assert.fail(EXCEPTION_PREFIX + "null number of containers"); + } catch (IllegalArgumentException e) { + Assert.assertTrue(e.getMessage() + .startsWith(ERROR_CONTAINERS_COUNT_INVALID)); + } + + // negative number of containers + app.setNumberOfContainers(-1L); + try { + ServiceApiUtil.validateApplicationPayload(app, sfs); + Assert.fail(EXCEPTION_PREFIX + "negative number of containers"); + } catch (IllegalArgumentException e) { + Assert.assertTrue(e.getMessage() + .startsWith(ERROR_CONTAINERS_COUNT_INVALID)); + } + + // everything valid here + app.setNumberOfContainers(5L); + try { + ServiceApiUtil.validateApplicationPayload(app, sfs); + } catch (IllegalArgumentException e) { + LOG.error("application attributes specified should be valid here", e); + Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage()); + } + + // no artifact id fails with default type + Artifact artifact = new Artifact(); + app.setArtifact(artifact); + try { + ServiceApiUtil.validateApplicationPayload(app, sfs); + Assert.fail(EXCEPTION_PREFIX + "application with no artifact id"); + } catch (IllegalArgumentException e) { + Assert.assertEquals(ERROR_ARTIFACT_ID_INVALID, e.getMessage()); + } + + // no artifact id fails with APPLICATION type + artifact.setType(Artifact.TypeEnum.APPLICATION); + try { + ServiceApiUtil.validateApplicationPayload(app, sfs); + Assert.fail(EXCEPTION_PREFIX + "application with no artifact id"); + } catch (IllegalArgumentException e) { + Assert.assertEquals(ERROR_ARTIFACT_ID_INVALID, e.getMessage()); + } + + // no artifact id fails with TARBALL type + artifact.setType(Artifact.TypeEnum.TARBALL); + try { + ServiceApiUtil.validateApplicationPayload(app, sfs); + Assert.fail(EXCEPTION_PREFIX + "application with no artifact id"); + } catch (IllegalArgumentException e) { + Assert.assertEquals(ERROR_ARTIFACT_ID_INVALID, e.getMessage()); + } + + // everything valid here + artifact.setType(Artifact.TypeEnum.DOCKER); + artifact.setId("docker.io/centos:centos7"); + try { + ServiceApiUtil.validateApplicationPayload(app, sfs); + } catch (IllegalArgumentException e) { + LOG.error("application attributes specified should be valid here", e); + Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage()); + } + + // defaults assigned + Assert.assertEquals(app.getComponents().get(0).getName(), + DEFAULT_COMPONENT_NAME); + Assert.assertEquals(app.getLifetime(), DEFAULT_UNLIMITED_LIFETIME); + } + + @Test + public void testConfigFiles() throws IOException { + FileSystem mockFs = createNiceMock(FileSystem.class); + expect(mockFs.exists(anyObject(Path.class))).andReturn(true).anyTimes(); + replay(mockFs); + + ConfigFile configFile = new ConfigFile(); + List configFiles = new ArrayList<>(); + configFiles.add(configFile); + + try { + ServiceApiUtil.validateConfigFile(configFiles, mockFs); + Assert.fail(EXCEPTION_PREFIX + "null file type"); + } catch (IllegalArgumentException e) { + } + + configFile.setType(TypeEnum.TEMPLATE); + try { + ServiceApiUtil.validateConfigFile(configFiles, mockFs); + Assert.fail(EXCEPTION_PREFIX + "empty src_file for type template"); + } catch (IllegalArgumentException e) { + } + + configFile.setSrcFile("srcfile"); + try { + ServiceApiUtil.validateConfigFile(configFiles, mockFs); + Assert.fail(EXCEPTION_PREFIX + "empty dest file"); + } catch (IllegalArgumentException e) { + } + + configFile.setDestFile("destfile"); + try { + ServiceApiUtil.validateConfigFile(configFiles, mockFs); + } catch (IllegalArgumentException e) { + Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage()); + } + + configFile = new ConfigFile(); + configFile.setType(TypeEnum.JSON); + configFile.setSrcFile(null); + configFile.setDestFile("destfile2"); + configFiles.add(configFile); + try { + ServiceApiUtil.validateConfigFile(configFiles, mockFs); + } catch (IllegalArgumentException e) { + Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage()); + } + + configFile.setDestFile("destfile"); + try { + ServiceApiUtil.validateConfigFile(configFiles, mockFs); + Assert.fail(EXCEPTION_PREFIX + "duplicate dest file"); + } catch (IllegalArgumentException e) { + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/YarnMiniClusterTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/YarnMiniClusterTestBase.java index 746a0ec..5e62fc2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/YarnMiniClusterTestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/YarnMiniClusterTestBase.java @@ -37,6 +37,7 @@ import org.apache.slider.common.SliderXmlConfKeys; import org.apache.slider.common.params.ActionFreezeArgs; import org.apache.slider.common.params.Arguments; +import org.apache.slider.common.params.SliderActions; import org.apache.slider.common.tools.Duration; import org.apache.slider.common.tools.SliderFileSystem; import org.apache.slider.common.tools.SliderUtils; @@ -328,11 +329,8 @@ protected YarnConfiguration getConfiguration() { */ public void stopRunningClusters() { for (SliderClient client : clustersToTeardown) { - try { - maybeStopCluster(client, "", "Teardown at end of test case", true); - } catch (Exception e) { - LOG.warn("While stopping cluster " + e, e); - } + maybeStopCluster(client, client.getDeployedClusterName(), + "Teardown at end of test case", true); } } @@ -502,6 +500,62 @@ protected String getFsDefaultName() { } /** + * Create or build a cluster (the action is set by the first verb). + * @param action operation to invoke: SliderActions.ACTION_CREATE or + * SliderActions.ACTION_BUILD + * @param clustername cluster name + * @param extraArgs list of extra args to add to the creation command + * @param deleteExistingData should the data of any existing cluster + * of this name be deleted + * @param blockUntilRunning block until the AM is running + * @return launcher which will have executed the command. + */ + public ServiceLauncher createOrBuildCluster(String action, + String clustername, List extraArgs, boolean deleteExistingData, + boolean blockUntilRunning) throws Throwable { + assertNotNull(clustername); + assertNotNull(miniCluster); + // update action should keep existing data + Configuration config = miniCluster.getConfig(); + if (deleteExistingData && !SliderActions.ACTION_UPDATE.equals(action)) { + FileSystem dfs = FileSystem.get(new URI(getFsDefaultName()), config); + + SliderFileSystem sliderFileSystem = new SliderFileSystem(dfs, config); + Path clusterDir = sliderFileSystem.buildClusterDirPath(clustername); + LOG.info("deleting instance data at {}", clusterDir); + //this is a safety check to stop us doing something stupid like deleting / + assertTrue(clusterDir.toString().contains("/.slider/")); + rigorousDelete(sliderFileSystem, clusterDir, 60000); + } + + + List argsList = new ArrayList<>(); + argsList.addAll(Arrays.asList( + action, clustername, + Arguments.ARG_MANAGER, getRMAddr(), + Arguments.ARG_FILESYSTEM, getFsDefaultName(), + Arguments.ARG_DEBUG)); + + argsList.addAll(getExtraCLIArgs()); + + if (extraArgs != null) { + argsList.addAll(extraArgs); + } + ServiceLauncher launcher = launchClientAgainstMiniMR( + //config includes RM binding info + new YarnConfiguration(config), + //varargs list of command line params + argsList + ); + assertEquals(0, launcher.getServiceExitCode()); + SliderClient client = launcher.getService(); + if (blockUntilRunning) { + client.monitorAppToRunning(new Duration(CLUSTER_GO_LIVE_TIME)); + } + return launcher; + } + + /** * Delete with some pauses and backoff; designed to handle slow delete * operation in windows. */ @@ -652,28 +706,6 @@ public String getApplicationHome() { return getTestConfiguration().getTrimmed(getApplicationHomeKey()); } - public List getImageCommands() { - if (switchToImageDeploy) { - // its an image that had better be defined - assertNotNull(getArchivePath()); - if (!imageIsRemote) { - // its not remote, so assert it exists - File f = new File(getArchivePath()); - assertTrue(f.exists()); - return Arrays.asList(Arguments.ARG_IMAGE, f.toURI().toString()); - } else { - assertNotNull(remoteImageURI); - - // if it is remote, then its whatever the archivePath property refers to - return Arrays.asList(Arguments.ARG_IMAGE, remoteImageURI.toString()); - } - } else { - assertNotNull(getApplicationHome()); - assertTrue(new File(getApplicationHome()).exists()); - return Arrays.asList(Arguments.ARG_APP_HOME, getApplicationHome()); - } - } - /** * Get the resource configuration dir in the source tree. * @@ -746,14 +778,23 @@ public int maybeStopCluster( SliderClient sliderClient, String clustername, String message, - boolean force) throws IOException, YarnException { + boolean force) { if (sliderClient != null) { if (SliderUtils.isUnset(clustername)) { clustername = sliderClient.getDeployedClusterName(); } //only stop a cluster that exists if (SliderUtils.isSet(clustername)) { - return clusterActionFreeze(sliderClient, clustername, message, force); + try { + clusterActionFreeze(sliderClient, clustername, message, force); + } catch (Exception e) { + LOG.warn("While stopping cluster " + e, e); + } + try { + sliderClient.actionDestroy(clustername); + } catch (Exception e) { + LOG.warn("While destroying cluster " + e, e); + } } } return 0; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/YarnZKMiniClusterTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/YarnZKMiniClusterTestBase.java index 322b346..cf9e616 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/YarnZKMiniClusterTestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/YarnZKMiniClusterTestBase.java @@ -22,7 +22,6 @@ import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.registry.client.api.RegistryConstants; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.slider.common.tools.SliderUtils; import org.apache.slider.core.zk.BlockingZKWatcher; import org.apache.slider.core.zk.ZKIntegration; import org.slf4j.Logger; @@ -109,9 +108,7 @@ protected String createMiniCluster(String name, int numLogDirs, boolean startZK, boolean startHDFS) throws IOException { - if (SliderUtils.isUnset(name)) { - name = methodName.getMethodName(); - } + name = buildClustername(name); createMicroZKCluster("-" + name, conf); conf.setBoolean(RegistryConstants.KEY_REGISTRY_ENABLED, true); conf.set(RegistryConstants.KEY_REGISTRY_ZK_QUORUM, getZKBinding()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app-override-resolved.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app-override-resolved.json deleted file mode 100644 index e2a21ea..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app-override-resolved.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "name": "app-1", - "lifetime": "3600", - "configuration": { - "properties": { - "g1": "a", - "g2": "b" - } - }, - "resource": { - "cpus": 1, - "memory": "512" - }, - "number_of_containers": 2, - "components": [ - { - "name": "simple", - "configuration": { - "properties": { - "g1": "a", - "g2": "b" - } - } - }, - { - "name": "master", - "configuration": { - "properties": { - "g1": "overridden", - "g2": "b" - } - } - }, - { - "name": "worker", - "resource": { - "cpus": 1, - "memory": "1024" - }, - "configuration": { - "properties": { - "g1": "overridden-by-worker", - "g2": "b", - "timeout": "1000" - } - } - } - ] -} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app-override.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app-override.json index 552cdef..19393bf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app-override.json +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app-override.json @@ -5,7 +5,24 @@ "properties": { "g1": "a", "g2": "b" - } + }, + "files": [ + { + "type": "PROPERTIES", + "dest_file": "file1", + "props": { + "k1": "v1", + "k2": "v2" + } + }, + { + "type": "XML", + "dest_file": "file2", + "props": { + "k3": "v3" + } + } + ] }, "resource": { "cpus": 1, @@ -14,7 +31,18 @@ "number_of_containers": 2, "components": [ { - "name": "simple" + "name": "simple", + "configuration": { + "files": [ + { + "type": "PROPERTIES", + "dest_file": "file1", + "props": { + "k1": "overridden" + } + } + ] + } }, { "name": "master", diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app-resolved.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app-resolved.json deleted file mode 100644 index cd1ab6f..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app-resolved.json +++ /dev/null @@ -1,81 +0,0 @@ -{ - "name": "zk-app-1", - "lifetime": "3600", - "configuration": { - "properties": { - "internal.chaos.monkey.interval.seconds": "60", - "zookeeper.port": "2181", - "zookeeper.path": "/yarnapps_small_cluster", - "zookeeper.hosts": "zoo1,zoo2,zoo3", - "env.MALLOC_ARENA_MAX": "4", - "site.hbase.master.startup.retainassign": "true", - "site.fs.defaultFS": "hdfs://cluster:8020", - "site.fs.default.name": "hdfs://cluster:8020", - "site.hbase.master.info.port": "0", - "site.hbase.regionserver.info.port": "0" - } - }, - "resource": { - "cpus": 1, - "memory": "512" - }, - "number_of_containers": 2, - "components": [ - { - "name": "simple", - "number_of_containers": 2, - "configuration": { - "properties": { - "g1": "a", - "g2": "b" - } - } - }, - { - "name": "master", - "number_of_containers": 1, - "resource": { - "cpus": 1, - "memory": "512" - }, - "configuration": { - "properties": { - "zookeeper.port": "2181", - "zookeeper.path": "/yarnapps_small_cluster", - "zookeeper.hosts": "zoo1,zoo2,zoo3", - "env.MALLOC_ARENA_MAX": "4", - "site.hbase.master.startup.retainassign": "true", - "site.fs.defaultFS": "hdfs://cluster:8020", - "site.fs.default.name": "hdfs://cluster:8020", - "site.hbase.master.info.port": "0", - "site.hbase.regionserver.info.port": "0", - "jvm.heapsize": "512M" - } - } - }, - { - "name": "worker", - "number_of_containers": 5, - "resource": { - "cpus": 1, - "memory": "1024" - }, - "configuration": { - "properties": { - "g1": "overridden-by-worker", - "g2": "b", - "zookeeper.port": "2181", - "zookeeper.path": "/yarnapps_small_cluster", - "zookeeper.hosts": "zoo1,zoo2,zoo3", - "env.MALLOC_ARENA_MAX": "4", - "site.hbase.master.startup.retainassign": "true", - "site.fs.defaultFS": "hdfs://cluster:8020", - "site.fs.default.name": "hdfs://cluster:8020", - "site.hbase.master.info.port": "0", - "site.hbase.regionserver.info.port": "0", - "jvm.heapsize": "512M" - } - } - } - ] -} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app.json index 90857db..7a7810c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app.json +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app.json @@ -5,16 +5,7 @@ "properties": { "g1": "a", "g2": "b", - "internal.chaos.monkey.interval.seconds": "60", - "zookeeper.port": "2181", - "zookeeper.path": "/yarnapps_small_cluster", - "zookeeper.hosts": "zoo1,zoo2,zoo3", - "env.MALLOC_ARENA_MAX": "4", - "site.hbase.master.startup.retainassign": "true", - "site.fs.defaultFS": "hdfs://cluster:8020", - "site.fs.default.name": "hdfs://cluster:8020", - "site.hbase.master.info.port": "0", - "site.hbase.regionserver.info.port": "0" + "internal.chaos.monkey.interval.seconds": "60" } }, "resource": { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/default.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/default.json new file mode 100644 index 0000000..16f0efc --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/default.json @@ -0,0 +1,16 @@ +{ + "name": "default-app-1", + "lifetime": "3600", + "components" : + [ + { + "name": "SLEEP", + "number_of_containers": 1, + "launch_command": "sleep 3600", + "resource": { + "cpus": 2, + "memory": "256" + } + } + ] +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/external0.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/external0.json new file mode 100644 index 0000000..1f9dfeb --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/external0.json @@ -0,0 +1,8 @@ +{ + "name": "external-0", + "lifetime": "3600", + "artifact": { + "type": "APPLICATION", + "id": "app-1" + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/external1.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/external1.json new file mode 100644 index 0000000..d7bf9bb --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/external1.json @@ -0,0 +1,21 @@ +{ + "name": "external-1", + "lifetime": "3600", + "components": [ + { + "name": "simple", + "artifact": { + "type": "APPLICATION", + "id": "app-1" + } + }, + { + "name": "other", + "number_of_containers": 2, + "resource": { + "cpus": 1, + "memory": "512" + } + } + ] +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/external2.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/external2.json new file mode 100644 index 0000000..c4337a1 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/external2.json @@ -0,0 +1,21 @@ +{ + "name": "external-2", + "lifetime": "3600", + "components": [ + { + "name": "ext", + "artifact": { + "type": "APPLICATION", + "id": "external-1" + } + }, + { + "name": "another", + "number_of_containers": 1, + "resource": { + "cpus": 1, + "memory": "512" + } + } + ] +}