diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
index 4e88aef..4c3d953 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
@@ -29,7 +29,6 @@
Hadoop YARN REST APIs for services
- false
1.6.5
@@ -81,30 +80,10 @@
org.apache.maven.plugins
maven-surefire-plugin
- ${maven-surefire-plugin.version}
- ${test.reuseForks}
- ${test.forkMode}
- 1
- ${test.forkedProcessTimeoutInSeconds}
-
- 1
- ${test.argLine}
- ${test.failIfNoTests}
- ${build.redirect.test.output.to.file}
- ${test.env.path}
+ ${java.home}
-
- true
- true
-
-
- **/Test*.java
-
-
- **/Test*$*.java
-
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/services/api/impl/TestApplicationApiService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/services/api/impl/TestApplicationApiService.java
index 6e077d2..22debd7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/services/api/impl/TestApplicationApiService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/services/api/impl/TestApplicationApiService.java
@@ -17,24 +17,10 @@
package org.apache.hadoop.yarn.services.api.impl;
-import static org.apache.slider.util.RestApiConstants.*;
-import static org.apache.slider.util.RestApiErrorMessages.*;
-
-import java.util.ArrayList;
-
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.api.resource.Artifact;
-import org.apache.slider.api.resource.Resource;
-import org.apache.slider.util.ServiceApiUtil;
-import org.junit.After;
-import org.junit.Assert;
import org.junit.Before;
-import org.junit.Test;
import org.junit.runner.RunWith;
import org.powermock.core.classloader.annotations.SuppressStaticInitializationFor;
import org.powermock.modules.junit4.PowerMockRunner;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
* Test class for application life time monitor feature test.
@@ -42,168 +28,14 @@
@RunWith(PowerMockRunner.class)
@SuppressStaticInitializationFor("org.apache.hadoop.yarn.services.api.impl.ApplicationApiService")
public class TestApplicationApiService {
- private static final Logger logger = LoggerFactory
- .getLogger(TestApplicationApiService.class);
- private static String EXCEPTION_PREFIX = "Should have thrown exception: ";
- private static String NO_EXCEPTION_PREFIX = "Should not have thrown exception: ";
private ApplicationApiService appApiService;
@Before
public void setup() throws Exception {
- appApiService = new ApplicationApiService();
+ appApiService = new ApplicationApiService();
}
- @After
- public void tearDown() throws Exception {
- }
-
- @Test(timeout = 90000)
- public void testValidateApplicationPostPayload() throws Exception {
- Application app = new Application();
-
- // no name
- try {
- ServiceApiUtil.validateApplicationPayload(app, null);
- Assert.fail(EXCEPTION_PREFIX + "application with no name");
- } catch (IllegalArgumentException e) {
- Assert.assertEquals(ERROR_APPLICATION_NAME_INVALID, e.getMessage());
- }
-
- // bad format name
- String[] badNames = { "4finance", "Finance", "finance@home" };
- for (String badName : badNames) {
- app.setName(badName);
- try {
- ServiceApiUtil.validateApplicationPayload(app, null);
- Assert.fail(EXCEPTION_PREFIX + "application with bad name " + badName);
- } catch (IllegalArgumentException e) {
- Assert.assertEquals(ERROR_APPLICATION_NAME_INVALID_FORMAT,
- e.getMessage());
- }
- }
-
- // no artifact
- app.setName("finance_home");
- try {
- ServiceApiUtil.validateApplicationPayload(app, null);
- Assert.fail(EXCEPTION_PREFIX + "application with no artifact");
- } catch (IllegalArgumentException e) {
- Assert.assertEquals(ERROR_ARTIFACT_INVALID, e.getMessage());
- }
-
- // no artifact id
- Artifact artifact = new Artifact();
- app.setArtifact(artifact);
- try {
- ServiceApiUtil.validateApplicationPayload(app, null);
- Assert.fail(EXCEPTION_PREFIX + "application with no artifact id");
- } catch (IllegalArgumentException e) {
- Assert.assertEquals(ERROR_ARTIFACT_ID_INVALID, e.getMessage());
- }
-
- // if artifact is of type APPLICATION then everything is valid here
- artifact.setType(Artifact.TypeEnum.APPLICATION);
- artifact.setId("app.io/hbase:facebook_0.2");
- app.setNumberOfContainers(5l);
- try {
- ServiceApiUtil.validateApplicationPayload(app, null);
- } catch (IllegalArgumentException e) {
- logger.error("application attributes specified should be valid here", e);
- Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
- }
-
- // default-component, default-lifetime and the property component_type
- // should get assigned here
- Assert.assertEquals(app.getComponents().get(0).getName(),
- DEFAULT_COMPONENT_NAME);
- Assert.assertEquals(app.getLifetime(), DEFAULT_UNLIMITED_LIFETIME);
- //TODO handle external app
-
- // unset artifact type, default component and no of containers to test other
- // validation logic
- artifact.setType(null);
- app.setComponents(new ArrayList<>());
- app.setNumberOfContainers(null);
-
- // resource not specified
- artifact.setId("docker.io/centos:centos7");
- try {
- ServiceApiUtil.validateApplicationPayload(app, null);
- Assert.fail(EXCEPTION_PREFIX + "application with no resource");
- } catch (IllegalArgumentException e) {
- Assert.assertEquals(ERROR_RESOURCE_INVALID, e.getMessage());
- }
-
- // memory not specified
- Resource res = new Resource();
- app.setResource(res);
- try {
- ServiceApiUtil.validateApplicationPayload(app, null);
- Assert.fail(EXCEPTION_PREFIX + "application with no memory");
- } catch (IllegalArgumentException e) {
- Assert.assertEquals(ERROR_RESOURCE_MEMORY_INVALID, e.getMessage());
- }
-
- // cpu does not need to be always specified, it's an optional feature in yarn
- // invalid no of cpus
- res.setMemory("100mb");
- res.setCpus(-2);
- try {
- ServiceApiUtil.validateApplicationPayload(app, null);
- Assert.fail(
- EXCEPTION_PREFIX + "application with invalid no of cpups");
- } catch (IllegalArgumentException e) {
- Assert.assertEquals(ERROR_RESOURCE_CPUS_INVALID_RANGE, e.getMessage());
- }
-
- // number of containers not specified
- res.setCpus(2);
- try {
- ServiceApiUtil.validateApplicationPayload(app, null);
- Assert.fail(
- EXCEPTION_PREFIX + "application with no container count");
- } catch (IllegalArgumentException e) {
- Assert.assertTrue(e.getMessage().contains(ERROR_CONTAINERS_COUNT_INVALID));
- }
-
- // specifying profile along with cpus/memory raises exception
- res.setProfile("hbase_finance_large");
- try {
- ServiceApiUtil.validateApplicationPayload(app, null);
- Assert.fail(EXCEPTION_PREFIX
- + "application with resource profile along with cpus/memory");
- } catch (IllegalArgumentException e) {
- Assert.assertEquals(ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_NOT_SUPPORTED,
- e.getMessage());
- }
-
- // currently resource profile alone is not supported.
- // TODO: remove the next test once it is supported.
- res.setCpus(null);
- res.setMemory(null);
- try {
- ServiceApiUtil.validateApplicationPayload(app, null);
- Assert.fail(EXCEPTION_PREFIX
- + "application with resource profile only - NOT SUPPORTED");
- } catch (IllegalArgumentException e) {
- Assert.assertEquals(ERROR_RESOURCE_PROFILE_NOT_SUPPORTED_YET,
- e.getMessage());
- }
-
- // unset profile here and add cpus/memory back
- res.setProfile(null);
- res.setCpus(2);
- res.setMemory("2gb");
-
- // everything valid here
- app.setNumberOfContainers(5l);
- try {
- ServiceApiUtil.validateApplicationPayload(app, null);
- } catch (IllegalArgumentException e) {
- logger.error("application attributes specified should be valid here", e);
- Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
- }
-
- // Now test with components
+ public void testServicesApi() throws Exception {
+ // TODO write test for API
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Component.java
index e7f3796..f73c305 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Component.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Component.java
@@ -50,9 +50,9 @@
private String name = null;
private List dependencies = new ArrayList();
private ReadinessCheck readinessCheck = null;
- private Artifact artifact = new Artifact();
+ private Artifact artifact = null;
private String launchCommand = null;
- private Resource resource = new Resource();
+ private Resource resource = null;
private Long numberOfContainers = null;
private Boolean uniqueComponentSupport = false;
private Boolean runPrivilegedContainer = false;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java
index 37d1a40..e89306c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java
@@ -22,6 +22,7 @@
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import org.apache.commons.lang.StringUtils;
+import org.apache.slider.common.tools.SliderUtils;
import java.io.Serializable;
import java.util.ArrayList;
@@ -197,8 +198,10 @@ private String toIndentedString(java.lang.Object o) {
* this ConfigFile.
*/
public synchronized void mergeFrom(Configuration that) {
- this.properties.putAll(that.getProperties());
- this.env.putAll(that.getEnv());
+ SliderUtils.mergeMapsIgnoreDuplicateKeys(this.properties, that
+ .getProperties());
+ SliderUtils.mergeMapsIgnoreDuplicateKeys(this.env, that.getEnv());
+
Map thatMap = new HashMap<>();
for (ConfigFile file : that.getFiles()) {
thatMap.put(file.getDestFile(), file.copy());
@@ -206,7 +209,8 @@ public synchronized void mergeFrom(Configuration that) {
for (ConfigFile thisFile : files) {
if(thatMap.containsKey(thisFile.getDestFile())) {
ConfigFile thatFile = thatMap.get(thisFile.getDestFile());
- thisFile.getProps().putAll(thatFile.getProps());
+ SliderUtils.mergeMapsIgnoreDuplicateKeys(thisFile.getProps(),
+ thatFile.getProps());
thatMap.remove(thisFile.getDestFile());
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 32d78b4..d57ca84 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -101,6 +101,7 @@
import org.apache.slider.common.params.ClientArgs;
import org.apache.slider.common.params.CommonArgs;
import org.apache.slider.common.tools.ConfigHelper;
+import org.apache.slider.common.tools.Duration;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.common.tools.SliderVersionInfo;
@@ -636,16 +637,17 @@ private Application getApplicationFromArgs(String clusterName,
public int actionBuild(Application application) throws YarnException,
IOException {
Path appDir = checkAppNotExistOnHdfs(application);
+ ServiceApiUtil.validateApplicationPayload(application, sliderFileSystem);
persistApp(appDir, application);
+ deployedClusterName = application.getName();
return EXIT_SUCCESS;
}
public ApplicationId actionCreate(Application application)
throws IOException, YarnException {
- ServiceApiUtil.validateApplicationPayload(application,
- sliderFileSystem.getFileSystem());
String appName = application.getName();
validateClusterName(appName);
+ ServiceApiUtil.validateApplicationPayload(application, sliderFileSystem);
verifyNoLiveApp(appName, "Create");
Path appDir = checkAppNotExistOnHdfs(application);
@@ -880,6 +882,14 @@ private Path checkAppNotExistOnHdfs(Application application)
return appDir;
}
+ private Path checkAppExistOnHdfs(String appName)
+ throws IOException, SliderException {
+ Path appDir = sliderFileSystem.buildClusterDirPath(appName);
+ sliderFileSystem.verifyPathExists(
+ new Path(appDir, appName + ".json"));
+ return appDir;
+ }
+
private void persistApp(Path appDir, Application application)
throws IOException, SliderException {
FsPermission appDirPermission = new FsPermission("750");
@@ -1802,23 +1812,24 @@ public int actionStop(String appName, ActionFreezeArgs freezeArgs)
public int actionStart(String appName, ActionThawArgs thaw)
throws YarnException, IOException {
validateClusterName(appName);
+ Path appDir = checkAppExistOnHdfs(appName);
+ Application application = ServiceApiUtil.loadApplication(sliderFileSystem,
+ appName);
+ ServiceApiUtil.validateApplicationPayload(application, sliderFileSystem);
// see if it is actually running and bail out;
verifyNoLiveApp(appName, "Thaw");
- Path appDir = sliderFileSystem.buildClusterDirPath(appName);
- Path appJson = new Path(appDir, appName + ".json");
- Application application =
- jsonSerDeser.load(sliderFileSystem.getFileSystem(), appJson);
- submitApp(application);
+ ApplicationId appId = submitApp(application);
+ application.setId(appId.toString());
+ // write app definition on to hdfs
+ persistApp(appDir, application);
return 0;
}
public Map flex(String appName, Map
componentCounts) throws YarnException, IOException {
validateClusterName(appName);
- Path appDir = sliderFileSystem.buildClusterDirPath(appName);
- Path appJson = new Path(appDir, appName + ".json");
- Application persistedApp =
- jsonSerDeser.load(sliderFileSystem.getFileSystem(), appJson);
+ Application persistedApp = ServiceApiUtil.loadApplication(sliderFileSystem,
+ appName);
Map original = new HashMap<>(componentCounts.size());
for (Component persistedComp : persistedApp.getComponents()) {
String name = persistedComp.getName();
@@ -1833,7 +1844,8 @@ public int actionStart(String appName, ActionThawArgs thaw)
+ " do not exist in app definition.");
}
jsonSerDeser
- .save(sliderFileSystem.getFileSystem(), appJson, persistedApp, true);
+ .save(sliderFileSystem.getFileSystem(), ServiceApiUtil.getAppJsonPath(
+ sliderFileSystem, appName), persistedApp, true);
log.info("Updated app definition file for components " + componentCounts
.keySet());
@@ -2705,6 +2717,18 @@ private int actionTokens(ActionTokensArgs args)
yarnClient);
}
+ @VisibleForTesting
+ public ApplicationReport monitorAppToRunning(Duration duration)
+ throws YarnException, IOException {
+ return monitorAppToState(YarnApplicationState.RUNNING, duration);
+ }
+
+ @VisibleForTesting
+ public ApplicationReport monitorAppToState(YarnApplicationState desiredState,
+ Duration duration)
+ throws YarnException, IOException {
+ return yarnClient.monitorAppToState(applicationId, desiredState, duration);
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
index 4839395..6f24691 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
@@ -245,4 +245,64 @@ public NodeInformationList listNodes(String label, boolean live)
}
return results;
}
+
+ /**
+ * Monitor the submitted application for reaching the requested state.
+ * Will also report if the app reaches a later state (failed, killed, etc)
+ * Kill application if duration!= null & time expires.
+ * @param appId Application Id of application to be monitored
+ * @param duration how long to wait -must be more than 0
+ * @param desiredState desired state.
+ * @return the application report -null on a timeout
+ * @throws YarnException
+ * @throws IOException
+ */
+ public ApplicationReport monitorAppToState(
+ ApplicationId appId, YarnApplicationState desiredState, Duration duration)
+ throws YarnException, IOException {
+
+ if (appId == null) {
+ throw new BadCommandArgumentsException("null application ID");
+ }
+ if (duration.limit <= 0) {
+ throw new BadCommandArgumentsException("Invalid monitoring duration");
+ }
+ log.debug("Waiting {} millis for app to reach state {} ",
+ duration.limit,
+ desiredState);
+ duration.start();
+ try {
+ while (true) {
+ // Get application report for the appId we are interested in
+
+ ApplicationReport r = getApplicationReport(appId);
+
+ log.debug("queried status is\n{}",
+ new SliderUtils.OnDemandReportStringifier(r));
+
+ YarnApplicationState state = r.getYarnApplicationState();
+ if (state.ordinal() >= desiredState.ordinal()) {
+ log.debug("App in desired state (or higher) :{}", state);
+ return r;
+ }
+ if (duration.getLimitExceeded()) {
+ log.debug(
+ "Wait limit of {} millis to get to state {}, exceeded; app status\n {}",
+ duration.limit,
+ desiredState,
+ new SliderUtils.OnDemandReportStringifier(r));
+ return null;
+ }
+
+ // sleep 1s.
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException ignored) {
+ log.debug("Thread sleep in monitoring loop interrupted");
+ }
+ }
+ } finally {
+ duration.close();
+ }
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index bc8e139..0241158 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -2540,14 +2540,4 @@ public static long getTimeRange(org.apache.slider.api.resource
long totalMinutes = days * 24 * 60 + hours * 24 + minutes;
return totalMinutes * 60 + seconds;
}
-
- public static void resolve(Application application) {
- org.apache.slider.api.resource.Configuration global = application
- .getConfiguration();
- for (Component component : application.getComponents()) {
- mergeMapsIgnoreDuplicateKeys(component.getConfiguration().getProperties(),
- global.getProperties());
- }
- // TODO merge other information to components
- }
-}
\ No newline at end of file
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 84dde08..06a4932 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -107,7 +107,6 @@
import org.apache.slider.core.main.LauncherExitCodes;
import org.apache.slider.core.main.RunService;
import org.apache.slider.core.main.ServiceLauncher;
-import org.apache.slider.core.persist.JsonSerDeser;
import org.apache.slider.core.registry.info.CustomRegistryConstants;
import org.apache.slider.providers.ProviderCompleted;
import org.apache.slider.providers.ProviderService;
@@ -157,6 +156,7 @@
import org.apache.slider.server.services.workflow.WorkflowExecutorService;
import org.apache.slider.server.services.workflow.WorkflowRpcService;
import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders;
+import org.apache.slider.util.ServiceApiUtil;
import org.codehaus.jackson.map.PropertyNamingStrategy;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -389,9 +389,6 @@
*/
private boolean securityEnabled;
private ContentCache contentCache;
- private static final JsonSerDeser jsonSerDeser =
- new JsonSerDeser(Application.class,
- PropertyNamingStrategy.CAMEL_CASE_TO_LOWER_CASE_WITH_UNDERSCORES);
/**
* resource limits
@@ -590,9 +587,7 @@ private int createAndRunCluster(String appName) throws Throwable {
Path appDir = new Path((serviceArgs.getAppDefDir()));
SliderFileSystem fs = getClusterFS();
fs.setAppDir(appDir);
- Path appJson = new Path(appDir, appName + ".json");
- log.info("Loading application definition from " + appJson);
- application = jsonSerDeser.load(fs.getFileSystem(), appJson);
+ application = ServiceApiUtil.loadApplication(fs, appName);
log.info("Application Json: " + application);
stateForProviders.setApplicationName(appName);
Configuration serviceConf = getConfig();
@@ -874,11 +869,6 @@ private int createAndRunCluster(String appName) throws Throwable {
scheduleFailureWindowResets(application.getConfiguration());
scheduleEscalation(application.getConfiguration());
- for (Component component : application.getComponents()) {
- // Merge app-level configuration into component level configuration
- component.getConfiguration().mergeFrom(application.getConfiguration());
- }
-
try {
// schedule YARN Registry registration
queue(new ActionRegisterServiceInstance(appName, appid, application));
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java
index ac89ed8..f72afd8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java
@@ -21,7 +21,7 @@
String ERROR_APPLICATION_NAME_INVALID =
"Application name is either empty or not provided";
String ERROR_APPLICATION_NAME_INVALID_FORMAT =
- "Application name is not valid - only lower case letters, digits,"
+ "Application name %s is not valid - only lower case letters, digits,"
+ " underscore and hyphen are allowed";
String ERROR_APPLICATION_NOT_RUNNING = "Application not running";
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/ServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/ServiceApiUtil.java
index d7c72a3..b2e6eb0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/ServiceApiUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/ServiceApiUtil.java
@@ -28,12 +28,17 @@
import org.apache.slider.api.resource.ConfigFile;
import org.apache.slider.api.resource.Configuration;
import org.apache.slider.api.resource.Resource;
+import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.persist.JsonSerDeser;
+import org.apache.slider.providers.AbstractClientProvider;
+import org.apache.slider.providers.SliderProviderFactory;
+import org.codehaus.jackson.map.PropertyNamingStrategy;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
-import java.nio.file.Paths;
+import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@@ -41,102 +46,138 @@
public class ServiceApiUtil {
private static final Logger log =
LoggerFactory.getLogger(ServiceApiUtil.class);
+ private static final JsonSerDeser jsonSerDeser =
+ new JsonSerDeser<>(Application.class,
+ PropertyNamingStrategy.CAMEL_CASE_TO_LOWER_CASE_WITH_UNDERSCORES);
+
@VisibleForTesting
public static void validateApplicationPayload(Application application,
- FileSystem fs) throws IOException {
+ SliderFileSystem fs) throws IOException {
if (StringUtils.isEmpty(application.getName())) {
throw new IllegalArgumentException(
RestApiErrorMessages.ERROR_APPLICATION_NAME_INVALID);
}
if (!SliderUtils.isClusternameValid(application.getName())) {
- throw new IllegalArgumentException(
- RestApiErrorMessages.ERROR_APPLICATION_NAME_INVALID_FORMAT);
+ throw new IllegalArgumentException(String.format(
+ RestApiErrorMessages.ERROR_APPLICATION_NAME_INVALID_FORMAT,
+ application.getName()));
}
// If the application has no components do top-level checks
if (!hasComponent(application)) {
- // artifact
- if (application.getArtifact() == null) {
- throw new IllegalArgumentException(
- RestApiErrorMessages.ERROR_ARTIFACT_INVALID);
- }
- if (StringUtils.isEmpty(application.getArtifact().getId())) {
- throw new IllegalArgumentException(
- RestApiErrorMessages.ERROR_ARTIFACT_ID_INVALID);
- }
+ // If artifact is of type APPLICATION, read other application components
+ if (application.getArtifact() != null && application.getArtifact()
+ .getType() == Artifact.TypeEnum.APPLICATION) {
+ if (StringUtils.isEmpty(application.getArtifact().getId())) {
+ throw new IllegalArgumentException(
+ RestApiErrorMessages.ERROR_ARTIFACT_ID_INVALID);
+ }
+ Application otherApplication = loadApplication(fs,
+ application.getArtifact().getId());
+ application.setComponents(otherApplication.getComponents());
+ application.setArtifact(null);
+ SliderUtils.mergeMapsIgnoreDuplicateKeys(application.getQuicklinks(),
+ otherApplication.getQuicklinks());
+ } else {
+ // artifact
+ AbstractClientProvider clientProvider = SliderProviderFactory
+ .getClientProvider(application.getArtifact());
+ clientProvider.validateArtifact(application.getArtifact(), fs
+ .getFileSystem());
- // If artifact is of type APPLICATION, add a slider specific property
- if (application.getArtifact().getType()
- == Artifact.TypeEnum.APPLICATION) {
- if (application.getConfiguration() == null) {
- application.setConfiguration(new Configuration());
+ // resource
+ validateApplicationResource(application.getResource(), null);
+
+ // container size
+ if (application.getNumberOfContainers() == null
+ || application.getNumberOfContainers() < 0) {
+ throw new IllegalArgumentException(
+ RestApiErrorMessages.ERROR_CONTAINERS_COUNT_INVALID + ": "
+ + application.getNumberOfContainers());
}
+ validateConfigFile(application.getConfiguration().getFiles(), fs
+ .getFileSystem());
+ clientProvider.validateConfigFiles(application.getConfiguration()
+ .getFiles(), fs.getFileSystem());
+ // Since it is a simple app with no components, create a default
+ // component
+ application.getComponents().add(createDefaultComponent(application));
}
- // resource
- validateApplicationResource(application.getResource(), null,
- application.getArtifact().getType());
-
- // container size
- if (application.getNumberOfContainers() == null
- || application.getNumberOfContainers() < 0) {
- throw new IllegalArgumentException(
- RestApiErrorMessages.ERROR_CONTAINERS_COUNT_INVALID + ": "
- + application.getNumberOfContainers());
+ }
+ // Now run checks for each component.
+ // Let global values take effect if component level values are not
+ // provided.
+ Artifact globalArtifact = application.getArtifact();
+ Resource globalResource = application.getResource();
+ Configuration globalConf = application.getConfiguration();
+ Long globalNumberOfContainers = application.getNumberOfContainers();
+ Set componentNames = new HashSet<>();
+ List componentsToRemove = new ArrayList<>();
+ List componentsToAdd = new ArrayList<>();
+ for (Component comp : application.getComponents()) {
+ if (componentNames.contains(comp.getName())) {
+ throw new IllegalArgumentException("Component name collision: " +
+ comp.getName());
}
- validateConfigFile(application.getConfiguration().getFiles(), fs);
- // Since it is a simple app with no components, create a default component
- application.getComponents().add(createDefaultComponent(application));
- } else {
- // If the application has components, then run checks for each component.
- // Let global values take effect if component level values are not
- // provided.
- Artifact globalArtifact = application.getArtifact();
- Resource globalResource = application.getResource();
- Long globalNumberOfContainers = application.getNumberOfContainers();
- for (Component comp : application.getComponents()) {
- // artifact
- if (comp.getArtifact() == null) {
- comp.setArtifact(globalArtifact);
- }
- // If still null raise validation exception
- if (comp.getArtifact() == null) {
- throw new IllegalArgumentException(String
- .format(RestApiErrorMessages.ERROR_ARTIFACT_FOR_COMP_INVALID,
- comp.getName()));
- }
+ // artifact
+ if (comp.getArtifact() == null) {
+ comp.setArtifact(globalArtifact);
+ }
+ // configuration
+ comp.getConfiguration().mergeFrom(globalConf);
+ // If artifact is of type APPLICATION, read other application components
+ if (comp.getArtifact() != null && comp.getArtifact().getType() ==
+ Artifact.TypeEnum.APPLICATION) {
if (StringUtils.isEmpty(comp.getArtifact().getId())) {
- throw new IllegalArgumentException(String
- .format(RestApiErrorMessages.ERROR_ARTIFACT_ID_FOR_COMP_INVALID,
- comp.getName()));
+ throw new IllegalArgumentException(
+ RestApiErrorMessages.ERROR_ARTIFACT_ID_INVALID);
}
-
- // If artifact is of type APPLICATION, add a slider specific property
- if (comp.getArtifact().getType() == Artifact.TypeEnum.APPLICATION) {
- if (comp.getConfiguration() == null) {
- comp.setConfiguration(new Configuration());
+ componentsToRemove.add(comp);
+ List applicationComponents = getApplicationComponents(fs,
+ comp.getArtifact().getId());
+ for (Component c : applicationComponents) {
+ if (componentNames.contains(c.getName())) {
+ // TODO allow name collisions? see AppState#roles
+ // TODO or add prefix to external component names?
+ throw new IllegalArgumentException("Component name collision: " +
+ c.getName());
}
- comp.setName(comp.getArtifact().getId());
+ componentNames.add(c.getName());
}
+ componentsToAdd.addAll(applicationComponents);
+ } else {
+ componentNames.add(comp.getName());
+ }
+ }
+ application.getComponents().removeAll(componentsToRemove);
+ application.getComponents().addAll(componentsToAdd);
- // resource
- if (comp.getResource() == null) {
- comp.setResource(globalResource);
- }
- validateApplicationResource(comp.getResource(), comp,
- comp.getArtifact().getType());
+ for (Component comp : application.getComponents()) {
+ AbstractClientProvider compClientProvider = SliderProviderFactory
+ .getClientProvider(comp.getArtifact());
+ compClientProvider.validateArtifact(comp.getArtifact(), fs
+ .getFileSystem());
- // container count
- if (comp.getNumberOfContainers() == null) {
- comp.setNumberOfContainers(globalNumberOfContainers);
- }
- if (comp.getNumberOfContainers() == null
- || comp.getNumberOfContainers() < 0) {
- throw new IllegalArgumentException(String.format(
- RestApiErrorMessages.ERROR_CONTAINERS_COUNT_FOR_COMP_INVALID
- + ": " + comp.getNumberOfContainers(), comp.getName()));
- }
- validateConfigFile(comp.getConfiguration().getFiles(), fs);
+ // resource
+ if (comp.getResource() == null) {
+ comp.setResource(globalResource);
+ }
+ validateApplicationResource(comp.getResource(), comp);
+
+ // container count
+ if (comp.getNumberOfContainers() == null) {
+ comp.setNumberOfContainers(globalNumberOfContainers);
+ }
+ if (comp.getNumberOfContainers() == null
+ || comp.getNumberOfContainers() < 0) {
+ throw new IllegalArgumentException(String.format(
+ RestApiErrorMessages.ERROR_CONTAINERS_COUNT_FOR_COMP_INVALID
+ + ": " + comp.getNumberOfContainers(), comp.getName()));
}
+ validateConfigFile(comp.getConfiguration().getFiles(), fs
+ .getFileSystem());
+ compClientProvider.validateConfigFiles(comp.getConfiguration()
+ .getFiles(), fs.getFileSystem());
}
// Application lifetime if not specified, is set to unlimited lifetime
@@ -145,13 +186,38 @@ public static void validateApplicationPayload(Application application,
}
}
+ @VisibleForTesting
+ public static List getApplicationComponents(SliderFileSystem
+ fs, String appName) throws IOException {
+ return loadApplication(fs, appName).getComponents();
+ }
+
+ public static Application loadApplication(SliderFileSystem fs, String
+ appName) throws IOException {
+ Path appJson = getAppJsonPath(fs, appName);
+ log.info("Loading application definition from " + appJson);
+ Application externalApplication = jsonSerDeser.load(fs
+ .getFileSystem(), appJson);
+ return externalApplication;
+ }
+
+ public static Path getAppJsonPath(SliderFileSystem fs, String appName) {
+ Path appDir = fs.buildClusterDirPath(appName);
+ Path appJson = new Path(appDir, appName + ".json");
+ return appJson;
+ }
+
// 1) Verify the src_file exists and non-empty for template
- // 2) dest_file is absolute path
- private static void validateConfigFile(List list, FileSystem fs)
+ @VisibleForTesting
+ public static void validateConfigFile(List list, FileSystem fs)
throws IOException {
Set destFileSet = new HashSet<>();
for (ConfigFile file : list) {
+ if (file.getType() == null) {
+ throw new IllegalArgumentException("File type is empty");
+ }
+
if (file.getType().equals(ConfigFile.TypeEnum.TEMPLATE) && StringUtils
.isEmpty(file.getSrcFile())) {
throw new IllegalArgumentException(
@@ -170,11 +236,6 @@ private static void validateConfigFile(List list, FileSystem fs)
if (StringUtils.isEmpty(file.getDestFile())) {
throw new IllegalArgumentException("Dest_file is empty.");
}
- // validate dest_file is absolute
- if (!Paths.get(file.getDestFile()).isAbsolute()) {
- throw new IllegalArgumentException(
- "Dest_file must be absolute path: " + file.getDestFile());
- }
if (destFileSet.contains(file.getDestFile())) {
throw new IllegalArgumentException(
@@ -186,11 +247,8 @@ private static void validateConfigFile(List list, FileSystem fs)
private static void validateApplicationResource(Resource resource,
- Component comp, Artifact.TypeEnum artifactType) {
+ Component comp) {
// Only apps/components of type APPLICATION can skip resource requirement
- if (resource == null && artifactType == Artifact.TypeEnum.APPLICATION) {
- return;
- }
if (resource == null) {
throw new IllegalArgumentException(
comp == null ? RestApiErrorMessages.ERROR_RESOURCE_INVALID : String
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestKeytabCommandOptions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestKeytabCommandOptions.java
index 07d8c10..59ccda7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestKeytabCommandOptions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestKeytabCommandOptions.java
@@ -34,6 +34,7 @@
import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.main.ServiceLauncher;
import org.apache.slider.utils.SliderTestBase;
+import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -50,6 +51,7 @@
public class TestKeytabCommandOptions extends SliderTestBase {
private static SliderFileSystem testFileSystem;
+ private File testFolderDir;
@Before
public void setupFilesystem() throws IOException {
@@ -57,11 +59,18 @@ public void setupFilesystem() throws IOException {
YarnConfiguration configuration = SliderUtils.createConfiguration();
fileSystem.setConf(configuration);
testFileSystem = new SliderFileSystem(fileSystem, configuration);
- File testFolderDir = new File(testFileSystem
+ testFolderDir = new File(testFileSystem
.buildKeytabInstallationDirPath("").toUri().getPath());
FileUtils.deleteDirectory(testFolderDir);
}
+ @After
+ public void cleanup() throws IOException {
+ if (testFolderDir != null && testFolderDir.exists()) {
+ FileUtils.deleteDirectory(testFolderDir);
+ }
+ }
+
@Test
public void testInstallKeytab() throws Throwable {
// create a mock keytab file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestMiscSliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestMiscSliderUtils.java
index bf6ee2c..3392ff8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestMiscSliderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestMiscSliderUtils.java
@@ -45,5 +45,7 @@ public void testPurgeTempDir() throws Throwable {
assertTrue(fs.exists(inst));
sliderFileSystem.purgeAppInstanceTempFiles(CLUSTER1);
assertFalse(fs.exists(inst));
+
+ fs.delete(sliderFileSystem.buildClusterDirPath(CLUSTER1), true);
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/ExampleConfResources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/ExampleConfResources.java
index f13fbcc..13beb9b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/ExampleConfResources.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/ExampleConfResources.java
@@ -32,15 +32,17 @@
public final class ExampleConfResources {
public static final String APP_JSON = "app.json";
- public static final String APP_RES = "app-resolved.json";
public static final String OVERRIDE_JSON = "app-override.json";
- public static final String OVERRIDE_RES = "app-override-resolved.json";
+ public static final String DEFAULT_JSON = "default.json";
+ public static final String EXTERNAL_JSON_0 = "external0.json";
+ public static final String EXTERNAL_JSON_1 = "external1.json";
+ public static final String EXTERNAL_JSON_2 = "external2.json";
public static final String PACKAGE = "/org/apache/slider/core/conf/examples/";
- private static final String[] ALL_EXAMPLES = {APP_JSON, APP_RES,
- OVERRIDE_JSON, OVERRIDE_RES};
+ private static final String[] ALL_EXAMPLES = {APP_JSON, OVERRIDE_JSON,
+ DEFAULT_JSON};
public static final List ALL_EXAMPLE_RESOURCES = new ArrayList<>();
static {
@@ -55,4 +57,8 @@ private ExampleConfResources() {
static Application loadResource(String name) throws IOException {
return JSON_SER_DESER.fromResource(PACKAGE + name);
}
+
+ public static String resourceName(String name) {
+ return "target/test-classes" + PACKAGE + name;
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestConfTreeLoadExamples.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestConfTreeLoadExamples.java
index 48b0736..bafbdfc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestConfTreeLoadExamples.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestConfTreeLoadExamples.java
@@ -18,8 +18,11 @@
package org.apache.slider.core.conf;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.slider.api.resource.Application;
-import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.util.ServiceApiUtil;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -29,6 +32,10 @@
import java.util.Collection;
import static org.apache.slider.utils.SliderTestUtils.JSON_SER_DESER;
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.createNiceMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
/**
* Test loading example resources.
@@ -56,7 +63,15 @@ public TestConfTreeLoadExamples(String resource) {
public void testLoadResource() throws Throwable {
try {
Application application = JSON_SER_DESER.fromResource(resource);
- SliderUtils.resolve(application);
+
+ SliderFileSystem sfs = createNiceMock(SliderFileSystem.class);
+ FileSystem mockFs = createNiceMock(FileSystem.class);
+ expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes();
+ expect(sfs.buildClusterDirPath(anyObject())).andReturn(
+ new Path("cluster_dir_path")).anyTimes();
+ replay(sfs, mockFs);
+
+ ServiceApiUtil.validateApplicationPayload(application, sfs);
} catch (Exception e) {
throw new Exception("exception loading " + resource + ":" + e.toString());
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestConfigurationResolve.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestConfigurationResolve.java
index 285ddfa..ac2cc2e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestConfigurationResolve.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestConfigurationResolve.java
@@ -18,20 +18,36 @@
package org.apache.slider.core.conf;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.slider.api.resource.Application;
+import org.apache.slider.api.resource.ConfigFile;
+import org.apache.slider.api.resource.ConfigFile.TypeEnum;
import org.apache.slider.api.resource.Configuration;
+import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.util.ServiceApiUtil;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
import static org.apache.slider.api.InternalKeys.CHAOS_MONKEY_INTERVAL;
import static org.apache.slider.api.InternalKeys.DEFAULT_CHAOS_MONKEY_INTERVAL_DAYS;
import static org.apache.slider.api.InternalKeys.DEFAULT_CHAOS_MONKEY_INTERVAL_HOURS;
import static org.apache.slider.api.InternalKeys.DEFAULT_CHAOS_MONKEY_INTERVAL_MINUTES;
import static org.apache.slider.core.conf.ExampleConfResources.APP_JSON;
import static org.apache.slider.core.conf.ExampleConfResources.OVERRIDE_JSON;
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.createNiceMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
/**
* Test global configuration resolution.
@@ -48,17 +64,21 @@ public void testOverride() throws Throwable {
Configuration global = orig.getConfiguration();
assertEquals("a", global.getProperty("g1"));
assertEquals("b", global.getProperty("g2"));
+ assertEquals(2, global.getFiles().size());
Configuration simple = orig.getComponent("simple").getConfiguration();
assertEquals(0, simple.getProperties().size());
+ assertEquals(1, simple.getFiles().size());
Configuration master = orig.getComponent("master").getConfiguration();
assertEquals("m", master.getProperty("name"));
assertEquals("overridden", master.getProperty("g1"));
+ assertEquals(0, master.getFiles().size());
Configuration worker = orig.getComponent("worker").getConfiguration();
LOG.info("worker = {}", worker);
assertEquals(3, worker.getProperties().size());
+ assertEquals(0, worker.getFiles().size());
assertEquals("worker", worker.getProperty("name"));
assertEquals("overridden-by-worker", worker.getProperty("g1"));
@@ -66,18 +86,36 @@ public void testOverride() throws Throwable {
assertEquals("1000", worker.getProperty("timeout"));
// here is the resolution
- SliderUtils.resolve(orig);
+ SliderFileSystem sfs = createNiceMock(SliderFileSystem.class);
+ FileSystem mockFs = createNiceMock(FileSystem.class);
+ expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes();
+ expect(sfs.buildClusterDirPath(anyObject())).andReturn(
+ new Path("cluster_dir_path")).anyTimes();
+ replay(sfs, mockFs);
+ ServiceApiUtil.validateApplicationPayload(orig, sfs);
global = orig.getConfiguration();
LOG.info("global = {}", global);
assertEquals("a", global.getProperty("g1"));
assertEquals("b", global.getProperty("g2"));
+ assertEquals(2, global.getFiles().size());
simple = orig.getComponent("simple").getConfiguration();
assertEquals(2, simple.getProperties().size());
assertEquals("a", simple.getProperty("g1"));
assertEquals("b", simple.getProperty("g2"));
-
+ assertEquals(2, simple.getFiles().size());
+
+ Set files = new HashSet<>();
+ Map props = new HashMap<>();
+ props.put("k1", "overridden");
+ props.put("k2", "v2");
+ files.add(new ConfigFile().destFile("file1").type(TypeEnum
+ .PROPERTIES).props(props));
+ files.add(new ConfigFile().destFile("file2").type(TypeEnum
+ .XML).props(Collections.singletonMap("k3", "v3")));
+ assertTrue(files.contains(simple.getFiles().get(0)));
+ assertTrue(files.contains(simple.getFiles().get(1)));
master = orig.getComponent("master").getConfiguration();
LOG.info("master = {}", master);
@@ -85,6 +123,17 @@ public void testOverride() throws Throwable {
assertEquals("m", master.getProperty("name"));
assertEquals("overridden", master.getProperty("g1"));
assertEquals("b", master.getProperty("g2"));
+ assertEquals(2, master.getFiles().size());
+
+ props.put("k1", "v1");
+ files.clear();
+ files.add(new ConfigFile().destFile("file1").type(TypeEnum
+ .PROPERTIES).props(props));
+ files.add(new ConfigFile().destFile("file2").type(TypeEnum
+ .XML).props(Collections.singletonMap("k3", "v3")));
+
+ assertTrue(files.contains(master.getFiles().get(0)));
+ assertTrue(files.contains(master.getFiles().get(1)));
worker = orig.getComponent("worker").getConfiguration();
LOG.info("worker = {}", worker);
@@ -94,12 +143,14 @@ public void testOverride() throws Throwable {
assertEquals("overridden-by-worker", worker.getProperty("g1"));
assertEquals("b", worker.getProperty("g2"));
assertEquals("1000", worker.getProperty("timeout"));
+ assertEquals(2, worker.getFiles().size());
+ assertTrue(files.contains(worker.getFiles().get(0)));
+ assertTrue(files.contains(worker.getFiles().get(1)));
}
@Test
public void testTimeIntervalLoading() throws Throwable {
-
Application orig = ExampleConfResources.loadResource(APP_JSON);
Configuration conf = orig.getConfiguration();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestBuildApplicationComponent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestBuildApplicationComponent.java
new file mode 100644
index 0000000..0b23815
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestBuildApplicationComponent.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers;
+
+import org.apache.slider.api.resource.Component;
+import org.apache.slider.client.SliderClient;
+import org.apache.slider.common.params.SliderActions;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.core.conf.ExampleConfResources;
+import org.apache.slider.core.main.ServiceLauncher;
+import org.apache.slider.util.ServiceApiUtil;
+import org.apache.slider.utils.YarnZKMiniClusterTestBase;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import static org.apache.slider.common.params.Arguments.ARG_APPDEF;
+
+public class TestBuildApplicationComponent extends YarnZKMiniClusterTestBase {
+
+ private static void checkComponentNames(List components,
+ Set names) {
+ assertEquals(names.size(), components.size());
+ for (Component comp : components) {
+ assertTrue(names.contains(comp.getName()));
+ }
+ }
+
+ public void buildAndCheckComponents(String appName, String appDef,
+ SliderFileSystem sfs, Set names) throws Throwable {
+ ServiceLauncher launcher = createOrBuildCluster(
+ SliderActions.ACTION_BUILD, appName, Arrays.asList(ARG_APPDEF,
+ ExampleConfResources.resourceName(appDef)), true, false);
+ SliderClient sliderClient = launcher.getService();
+ addToTeardown(sliderClient);
+
+ // verify the cluster exists
+ assertEquals(0, sliderClient.actionExists(appName, false));
+ // verify generated conf
+ List components = ServiceApiUtil.getApplicationComponents(sfs,
+ appName);
+ checkComponentNames(components, names);
+ }
+
+ @Test
+ public void testExternalComponentBuild() throws Throwable {
+ String clustername = createMiniCluster("", getConfiguration(), 1, true);
+
+ describe("verify external components");
+
+ SliderFileSystem sfs = createSliderFileSystem();
+
+ Set nameSet = new HashSet<>();
+ nameSet.add("simple");
+ nameSet.add("master");
+ nameSet.add("worker");
+
+ buildAndCheckComponents("app-1", ExampleConfResources.APP_JSON, sfs,
+ nameSet);
+ buildAndCheckComponents("external-0", ExampleConfResources
+ .EXTERNAL_JSON_0, sfs, nameSet);
+
+ nameSet.add("other");
+
+ buildAndCheckComponents("external-1", ExampleConfResources
+ .EXTERNAL_JSON_1, sfs, nameSet);
+
+ nameSet.add("another");
+
+ buildAndCheckComponents("external-2", ExampleConfResources
+ .EXTERNAL_JSON_2, sfs, nameSet);
+
+ }
+
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestDefaultProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestDefaultProvider.java
new file mode 100644
index 0000000..2b59de9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestDefaultProvider.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers;
+
+import org.apache.slider.api.resource.Application;
+import org.apache.slider.client.SliderClient;
+import org.apache.slider.common.params.SliderActions;
+import org.apache.slider.core.conf.ExampleConfResources;
+import org.apache.slider.core.main.ServiceLauncher;
+import org.apache.slider.utils.YarnZKMiniClusterTestBase;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.util.Arrays;
+
+import static org.apache.slider.common.params.Arguments.ARG_APPDEF;
+
+public class TestDefaultProvider extends YarnZKMiniClusterTestBase {
+
+ // TODO figure out how to run client commands against minicluster
+ // (currently errors out unable to find containing jar of AM for upload)
+ @Ignore
+ @Test
+ public void testDefaultProvider() throws Throwable {
+ createMiniCluster("", getConfiguration(), 1, true);
+ String appName = "default-1";
+
+ describe("verify default provider");
+
+ String appDef = ExampleConfResources.resourceName(ExampleConfResources
+ .DEFAULT_JSON);
+
+ ServiceLauncher launcher = createOrBuildCluster(
+ SliderActions.ACTION_CREATE, appName, Arrays.asList(ARG_APPDEF,
+ appDef), true, true);
+ SliderClient sliderClient = launcher.getService();
+ addToTeardown(sliderClient);
+
+ Application application = sliderClient.actionStatus(appName);
+ assertEquals(1L, application.getContainers().size());
+ }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/BaseMockAppStateAATest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/BaseMockAppStateAATest.java
index c1f2886..6f4ca42 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/BaseMockAppStateAATest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/BaseMockAppStateAATest.java
@@ -43,7 +43,7 @@
@Override
public Application buildApplication() {
Application application = factory.newApplication(0, 0, 0)
- .name(getTestName());
+ .name(getValidTestName());
application.getComponent(ROLE1).getConfiguration().setProperty(
COMPONENT_PLACEMENT_POLICY, Integer.toString(PlacementPolicy
.ANTI_AFFINITY_REQUIRED));
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.java
index eb25b40..571e9d9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.java
@@ -362,7 +362,7 @@ public void testAMRestart() throws Throwable {
// now destroy the app state
AppStateBindingInfo bindingInfo = buildBindingInfo();
bindingInfo.application = factory.newApplication(0, 0, desiredAA).name(
- getTestName());
+ getValidTestName());
bindingInfo.application.getComponent(ROLE2)
.getConfiguration().setProperty(COMPONENT_PLACEMENT_POLICY,
Integer.toString(PlacementPolicy.ANTI_AFFINITY_REQUIRED));
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.java
index ea0dcf4..9cbda4f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.java
@@ -203,7 +203,7 @@ public void testRecurrentStartupFailureWithUnlimitedFailures() throws
// Update instance definition to allow containers to fail any number of
// times
AppStateBindingInfo bindingInfo = buildBindingInfo();
- bindingInfo.application.getConfiguration().setProperty(
+ bindingInfo.application.getComponent(ROLE0).getConfiguration().setProperty(
ResourceKeys.CONTAINER_FAILURE_THRESHOLD, "0");
appState = new MockAppState(bindingInfo);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.java
index 6d8e963..7f7f93a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.java
@@ -38,6 +38,7 @@
import org.slf4j.LoggerFactory;
import java.io.File;
+import java.io.IOException;
import java.util.Collections;
/**
@@ -65,7 +66,7 @@ public MockYarnEngine createYarnEngine() {
}
@Override
- public AppStateBindingInfo buildBindingInfo() {
+ public AppStateBindingInfo buildBindingInfo() throws IOException {
AppStateBindingInfo bindingInfo = super.buildBindingInfo();
bindingInfo.releaseSelector = new MostRecentContainerReleaseSelector();
return bindingInfo;
@@ -145,7 +146,7 @@ public void testHistoryFlexSaveResetLoad() throws Throwable {
appState = new MockAppState();
AppStateBindingInfo binding2 = buildBindingInfo();
binding2.application = factory.newApplication(0, 0, 0)
- .name(getTestName());
+ .name(getValidTestName());
binding2.historyPath = historyPath2;
appState.buildInstance(binding2);
// on this read there won't be the right number of roles
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.java
index b0634bf..d9c675d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.java
@@ -70,7 +70,7 @@ public void testRebuild() throws Throwable {
AppStateBindingInfo bindingInfo = buildBindingInfo();
bindingInfo.application = factory.newApplication(r0, r1, r2)
- .name(getTestName());
+ .name(getValidTestName());
bindingInfo.liveContainers = containers;
appState = new MockAppState(bindingInfo);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java
index b7e967f..703d65f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java
@@ -30,6 +30,7 @@
import org.apache.slider.server.appmaster.state.RoleStatus;
import org.junit.Test;
+import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.Map;
@@ -59,7 +60,7 @@ public MockYarnEngine createYarnEngine() {
}
@Override
- public AppStateBindingInfo buildBindingInfo() {
+ public AppStateBindingInfo buildBindingInfo() throws IOException {
AppStateBindingInfo bindingInfo = super.buildBindingInfo();
bindingInfo.releaseSelector = new MostRecentContainerReleaseSelector();
return bindingInfo;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.java
index d382c8a..4aa5895 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.java
@@ -40,7 +40,7 @@
@Override
public Application buildApplication() {
- return factory.newApplication(1, 0, 0).name(getTestName());
+ return factory.newApplication(1, 0, 0).name(getValidTestName());
}
@Test
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java
index 69abccf..6a2bd2d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java
@@ -51,6 +51,7 @@
import org.apache.slider.server.appmaster.state.RoleInstance;
import org.apache.slider.server.appmaster.state.RoleStatus;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
+import org.apache.slider.util.ServiceApiUtil;
import org.apache.slider.utils.SliderTestBase;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -62,6 +63,7 @@
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import java.util.Locale;
import java.util.Map.Entry;
/**
@@ -118,7 +120,7 @@ protected void initApp()
historyPath = new Path(historyWorkDir.toURI());
fs.delete(historyPath, true);
appState = new MockAppState(buildBindingInfo());
- stateAccess = new ProviderAppState(getTestName(), appState);
+ stateAccess = new ProviderAppState(getValidTestName(), appState);
}
/**
@@ -127,9 +129,11 @@ protected void initApp()
* from {@link #buildApplication()} ()}
* @return
*/
- protected AppStateBindingInfo buildBindingInfo() {
+ protected AppStateBindingInfo buildBindingInfo() throws IOException {
AppStateBindingInfo binding = new AppStateBindingInfo();
binding.application = buildApplication();
+ ServiceApiUtil.validateApplicationPayload(binding.application,
+ sliderFileSystem);
//binding.roles = new ArrayList<>(factory.ROLES);
binding.fs = fs;
binding.historyPath = historyPath;
@@ -142,7 +146,7 @@ protected AppStateBindingInfo buildBindingInfo() {
* @return the instance definition
*/
public Application buildApplication() {
- return factory.newApplication(0, 0, 0).name(getTestName());
+ return factory.newApplication(0, 0, 0).name(getValidTestName());
}
/**
@@ -153,6 +157,10 @@ public String getTestName() {
return methodName.getMethodName();
}
+ public String getValidTestName() {
+ return getTestName().toLowerCase(Locale.ENGLISH);
+ }
+
public RoleStatus getRole0Status() {
return lookupRole(ROLE0);
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockFactory.java
index 2ac5087..d27cd33 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockFactory.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockFactory.java
@@ -32,6 +32,7 @@
import org.apache.slider.api.ResourceKeys;
import org.apache.slider.api.resource.Application;
import org.apache.slider.api.resource.Component;
+import org.apache.slider.api.resource.Resource;
import org.apache.slider.providers.PlacementPolicy;
import org.apache.slider.providers.ProviderRole;
@@ -190,6 +191,7 @@ public MockContainer newContainer(AMRMClient.ContainerRequest req, String
*/
public Application newApplication(long r1, long r2, long r3) {
Application application = new Application();
+ application.setResource(new Resource().memory("256"));
application.getConfiguration().setProperty(ResourceKeys
.NODE_FAILURE_THRESHOLD, Integer.toString(NODE_FAILURE_THRESHOLD));
List components = application.getComponents();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestServiceApiUtil.java
new file mode 100644
index 0000000..c481c43
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestServiceApiUtil.java
@@ -0,0 +1,296 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.utils;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.slider.api.resource.Application;
+import org.apache.slider.api.resource.Artifact;
+import org.apache.slider.api.resource.ConfigFile;
+import org.apache.slider.api.resource.ConfigFile.TypeEnum;
+import org.apache.slider.api.resource.Resource;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.util.ServiceApiUtil;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.slider.util.RestApiConstants.DEFAULT_COMPONENT_NAME;
+import static org.apache.slider.util.RestApiConstants.DEFAULT_UNLIMITED_LIFETIME;
+import static org.apache.slider.util.RestApiErrorMessages.*;
+import static org.apache.slider.util.RestApiErrorMessages.ERROR_CONTAINERS_COUNT_INVALID;
+import static org.apache.slider.util.RestApiErrorMessages.ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_NOT_SUPPORTED;
+import static org.apache.slider.util.RestApiErrorMessages.ERROR_RESOURCE_PROFILE_NOT_SUPPORTED_YET;
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.createNiceMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+
+public class TestServiceApiUtil {
+ private static final Logger logger = LoggerFactory
+ .getLogger(TestServiceApiUtil.class);
+ private static String EXCEPTION_PREFIX = "Should have thrown exception: ";
+ private static String NO_EXCEPTION_PREFIX = "Should not have thrown exception: ";
+
+ @Test(timeout = 90000)
+ public void testValidateApplicationPostPayload() throws Exception {
+ SliderFileSystem sfs = createNiceMock(SliderFileSystem.class);
+ FileSystem mockFs = createNiceMock(FileSystem.class);
+ expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes();
+ expect(sfs.buildClusterDirPath(anyObject())).andReturn(
+ new Path("cluster_dir_path")).anyTimes();
+ replay(sfs, mockFs);
+
+ Application app = new Application();
+
+ // no name
+ try {
+ ServiceApiUtil.validateApplicationPayload(app, sfs);
+ Assert.fail(EXCEPTION_PREFIX + "application with no name");
+ } catch (IllegalArgumentException e) {
+ Assert.assertEquals(ERROR_APPLICATION_NAME_INVALID, e.getMessage());
+ }
+
+ // bad format name
+ String[] badNames = { "4finance", "Finance", "finance@home" };
+ for (String badName : badNames) {
+ app.setName(badName);
+ try {
+ ServiceApiUtil.validateApplicationPayload(app, sfs);
+ Assert.fail(EXCEPTION_PREFIX + "application with bad name " + badName);
+ } catch (IllegalArgumentException e) {
+ Assert.assertEquals(String.format(
+ ERROR_APPLICATION_NAME_INVALID_FORMAT, badName),
+ e.getMessage());
+ }
+ }
+
+ // resource not specified
+ app.setName("finance_home");
+ try {
+ ServiceApiUtil.validateApplicationPayload(app, sfs);
+ Assert.fail(EXCEPTION_PREFIX + "application with no resource");
+ } catch (IllegalArgumentException e) {
+ Assert.assertEquals(ERROR_RESOURCE_INVALID, e.getMessage());
+ }
+
+ // memory not specified
+ Resource res = new Resource();
+ app.setResource(res);
+ try {
+ ServiceApiUtil.validateApplicationPayload(app, sfs);
+ Assert.fail(EXCEPTION_PREFIX + "application with no memory");
+ } catch (IllegalArgumentException e) {
+ Assert.assertEquals(ERROR_RESOURCE_MEMORY_INVALID, e.getMessage());
+ }
+
+ // cpu does not need to be always specified, it's an optional feature in yarn
+ // invalid no of cpus
+ res.setMemory("100mb");
+ res.setCpus(-2);
+ try {
+ ServiceApiUtil.validateApplicationPayload(app, sfs);
+ Assert.fail(
+ EXCEPTION_PREFIX + "application with invalid no of cpups");
+ } catch (IllegalArgumentException e) {
+ Assert.assertEquals(ERROR_RESOURCE_CPUS_INVALID_RANGE, e.getMessage());
+ }
+
+ // number of containers not specified
+ res.setCpus(2);
+ try {
+ ServiceApiUtil.validateApplicationPayload(app, sfs);
+ Assert.fail(
+ EXCEPTION_PREFIX + "application with no container count");
+ } catch (IllegalArgumentException e) {
+ Assert.assertTrue(e.getMessage().contains(ERROR_CONTAINERS_COUNT_INVALID));
+ }
+
+ // specifying profile along with cpus/memory raises exception
+ res.setProfile("hbase_finance_large");
+ try {
+ ServiceApiUtil.validateApplicationPayload(app, sfs);
+ Assert.fail(EXCEPTION_PREFIX
+ + "application with resource profile along with cpus/memory");
+ } catch (IllegalArgumentException e) {
+ Assert.assertEquals(ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_NOT_SUPPORTED,
+ e.getMessage());
+ }
+
+ // currently resource profile alone is not supported.
+ // TODO: remove the next test once it is supported.
+ res.setCpus(null);
+ res.setMemory(null);
+ try {
+ ServiceApiUtil.validateApplicationPayload(app, sfs);
+ Assert.fail(EXCEPTION_PREFIX
+ + "application with resource profile only - NOT SUPPORTED");
+ } catch (IllegalArgumentException e) {
+ Assert.assertEquals(ERROR_RESOURCE_PROFILE_NOT_SUPPORTED_YET,
+ e.getMessage());
+ }
+
+ // unset profile here and add cpus/memory back
+ res.setProfile(null);
+ res.setCpus(2);
+ res.setMemory("2gb");
+
+ // null number of containers
+ try {
+ ServiceApiUtil.validateApplicationPayload(app, sfs);
+ Assert.fail(EXCEPTION_PREFIX + "null number of containers");
+ } catch (IllegalArgumentException e) {
+ Assert.assertTrue(e.getMessage()
+ .startsWith(ERROR_CONTAINERS_COUNT_INVALID));
+ }
+
+ // negative number of containers
+ app.setNumberOfContainers(-1L);
+ try {
+ ServiceApiUtil.validateApplicationPayload(app, sfs);
+ Assert.fail(EXCEPTION_PREFIX + "negative number of containers");
+ } catch (IllegalArgumentException e) {
+ Assert.assertTrue(e.getMessage()
+ .startsWith(ERROR_CONTAINERS_COUNT_INVALID));
+ }
+
+ // everything valid here
+ app.setNumberOfContainers(5l);
+ try {
+ ServiceApiUtil.validateApplicationPayload(app, sfs);
+ } catch (IllegalArgumentException e) {
+ logger.error("application attributes specified should be valid here", e);
+ Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
+ }
+
+ // no artifact id fails with default type
+ Artifact artifact = new Artifact();
+ app.setArtifact(artifact);
+ try {
+ ServiceApiUtil.validateApplicationPayload(app, sfs);
+ Assert.fail(EXCEPTION_PREFIX + "application with no artifact id");
+ } catch (IllegalArgumentException e) {
+ Assert.assertEquals(ERROR_ARTIFACT_ID_INVALID, e.getMessage());
+ }
+
+ // no artifact id fails with APPLICATION type
+ artifact.setType(Artifact.TypeEnum.APPLICATION);
+ try {
+ ServiceApiUtil.validateApplicationPayload(app, sfs);
+ Assert.fail(EXCEPTION_PREFIX + "application with no artifact id");
+ } catch (IllegalArgumentException e) {
+ Assert.assertEquals(ERROR_ARTIFACT_ID_INVALID, e.getMessage());
+ }
+
+ // no artifact id fails with TARBALL type
+ artifact.setType(Artifact.TypeEnum.TARBALL);
+ try {
+ ServiceApiUtil.validateApplicationPayload(app, sfs);
+ Assert.fail(EXCEPTION_PREFIX + "application with no artifact id");
+ } catch (IllegalArgumentException e) {
+ Assert.assertEquals(ERROR_ARTIFACT_ID_INVALID, e.getMessage());
+ }
+
+ // everything valid here
+ artifact.setType(Artifact.TypeEnum.DOCKER);
+ artifact.setId("docker.io/centos:centos7");
+ try {
+ ServiceApiUtil.validateApplicationPayload(app, sfs);
+ } catch (IllegalArgumentException e) {
+ logger.error("application attributes specified should be valid here", e);
+ Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
+ }
+
+ // default-component, default-lifetime and the property component_type
+ // should get assigned here
+ Assert.assertEquals(app.getComponents().get(0).getName(),
+ DEFAULT_COMPONENT_NAME);
+ Assert.assertEquals(app.getLifetime(), DEFAULT_UNLIMITED_LIFETIME);
+ //TODO handle external app
+
+ // unset artifact type, default component and no of containers to test other
+ // validation logic
+ artifact.setType(null);
+ app.setComponents(new ArrayList<>());
+ app.setNumberOfContainers(null);
+
+
+ // Now test with components
+ }
+
+ @Test
+ public void testConfigFiles() throws IOException {
+ FileSystem mockFs = createNiceMock(FileSystem.class);
+ expect(mockFs.exists(anyObject(Path.class))).andReturn(true).anyTimes();
+ replay(mockFs);
+
+ ConfigFile configFile = new ConfigFile();
+ List configFiles = new ArrayList<>();
+ configFiles.add(configFile);
+
+ try {
+ ServiceApiUtil.validateConfigFile(configFiles, mockFs);
+ Assert.fail(EXCEPTION_PREFIX + "null file type");
+ } catch (IllegalArgumentException e) {
+ }
+
+ configFile.setType(TypeEnum.TEMPLATE);
+ try {
+ ServiceApiUtil.validateConfigFile(configFiles, mockFs);
+ Assert.fail(EXCEPTION_PREFIX + "empty src_file for type template");
+ } catch (IllegalArgumentException e) {
+ }
+
+ configFile.setSrcFile("srcfile");
+ try {
+ ServiceApiUtil.validateConfigFile(configFiles, mockFs);
+ Assert.fail(EXCEPTION_PREFIX + "empty dest file");
+ } catch (IllegalArgumentException e) {
+ }
+
+ configFile.setDestFile("destfile");
+ try {
+ ServiceApiUtil.validateConfigFile(configFiles, mockFs);
+ } catch (IllegalArgumentException e) {
+ Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
+ }
+
+ configFile = new ConfigFile();
+ configFile.setType(TypeEnum.JSON);
+ configFile.setSrcFile(null);
+ configFile.setDestFile("destfile2");
+ configFiles.add(configFile);
+ try {
+ ServiceApiUtil.validateConfigFile(configFiles, mockFs);
+ } catch (IllegalArgumentException e) {
+ Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
+ }
+
+ configFile.setDestFile("destfile");
+ try {
+ ServiceApiUtil.validateConfigFile(configFiles, mockFs);
+ Assert.fail(EXCEPTION_PREFIX + "duplicate dest file");
+ } catch (IllegalArgumentException e) {
+ }
+ }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/YarnMiniClusterTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/YarnMiniClusterTestBase.java
index 746a0ec..df786c9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/YarnMiniClusterTestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/YarnMiniClusterTestBase.java
@@ -37,6 +37,7 @@
import org.apache.slider.common.SliderXmlConfKeys;
import org.apache.slider.common.params.ActionFreezeArgs;
import org.apache.slider.common.params.Arguments;
+import org.apache.slider.common.params.SliderActions;
import org.apache.slider.common.tools.Duration;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.common.tools.SliderUtils;
@@ -328,11 +329,8 @@ protected YarnConfiguration getConfiguration() {
*/
public void stopRunningClusters() {
for (SliderClient client : clustersToTeardown) {
- try {
- maybeStopCluster(client, "", "Teardown at end of test case", true);
- } catch (Exception e) {
- LOG.warn("While stopping cluster " + e, e);
- }
+ maybeStopCluster(client, client.getDeployedClusterName(),
+ "Teardown at end of test case", true);
}
}
@@ -502,6 +500,62 @@ protected String getFsDefaultName() {
}
/**
+ * Create or build a cluster (the action is set by the first verb)
+ * @param action operation to invoke: SliderActions.ACTION_CREATE or
+ * SliderActions.ACTION_BUILD
+ * @param clustername cluster name
+ * @param extraArgs list of extra args to add to the creation command
+ * @param deleteExistingData should the data of any existing cluster
+ * of this name be deleted
+ * @param blockUntilRunning block until the AM is running
+ * @return launcher which will have executed the command.
+ */
+ public ServiceLauncher createOrBuildCluster(String action,
+ String clustername, List extraArgs, boolean deleteExistingData,
+ boolean blockUntilRunning) throws Throwable {
+ assertNotNull(clustername);
+ assertNotNull(miniCluster);
+ // update action should keep existing data
+ Configuration config = miniCluster.getConfig();
+ if (deleteExistingData && !SliderActions.ACTION_UPDATE.equals(action)) {
+ FileSystem dfs = FileSystem.get(new URI(getFsDefaultName()), config);
+
+ SliderFileSystem sliderFileSystem = new SliderFileSystem(dfs, config);
+ Path clusterDir = sliderFileSystem.buildClusterDirPath(clustername);
+ LOG.info("deleting instance data at {}", clusterDir);
+ //this is a safety check to stop us doing something stupid like deleting /
+ assertTrue(clusterDir.toString().contains("/.slider/"));
+ rigorousDelete(sliderFileSystem, clusterDir, 60000);
+ }
+
+
+ List argsList = new ArrayList<>();
+ argsList.addAll(Arrays.asList(
+ action, clustername,
+ Arguments.ARG_MANAGER, getRMAddr(),
+ Arguments.ARG_FILESYSTEM, getFsDefaultName(),
+ Arguments.ARG_DEBUG));
+
+ argsList.addAll(getExtraCLIArgs());
+
+ if (extraArgs != null) {
+ argsList.addAll(extraArgs);
+ }
+ ServiceLauncher launcher = launchClientAgainstMiniMR(
+ //config includes RM binding info
+ new YarnConfiguration(config),
+ //varargs list of command line params
+ argsList
+ );
+ assertEquals(0, launcher.getServiceExitCode());
+ SliderClient client = launcher.getService();
+ if (blockUntilRunning) {
+ client.monitorAppToRunning(new Duration(CLUSTER_GO_LIVE_TIME));
+ }
+ return launcher;
+ }
+
+ /**
* Delete with some pauses and backoff; designed to handle slow delete
* operation in windows.
*/
@@ -652,28 +706,6 @@ public String getApplicationHome() {
return getTestConfiguration().getTrimmed(getApplicationHomeKey());
}
- public List getImageCommands() {
- if (switchToImageDeploy) {
- // its an image that had better be defined
- assertNotNull(getArchivePath());
- if (!imageIsRemote) {
- // its not remote, so assert it exists
- File f = new File(getArchivePath());
- assertTrue(f.exists());
- return Arrays.asList(Arguments.ARG_IMAGE, f.toURI().toString());
- } else {
- assertNotNull(remoteImageURI);
-
- // if it is remote, then its whatever the archivePath property refers to
- return Arrays.asList(Arguments.ARG_IMAGE, remoteImageURI.toString());
- }
- } else {
- assertNotNull(getApplicationHome());
- assertTrue(new File(getApplicationHome()).exists());
- return Arrays.asList(Arguments.ARG_APP_HOME, getApplicationHome());
- }
- }
-
/**
* Get the resource configuration dir in the source tree.
*
@@ -746,14 +778,23 @@ public int maybeStopCluster(
SliderClient sliderClient,
String clustername,
String message,
- boolean force) throws IOException, YarnException {
+ boolean force) {
if (sliderClient != null) {
if (SliderUtils.isUnset(clustername)) {
clustername = sliderClient.getDeployedClusterName();
}
//only stop a cluster that exists
if (SliderUtils.isSet(clustername)) {
- return clusterActionFreeze(sliderClient, clustername, message, force);
+ try {
+ clusterActionFreeze(sliderClient, clustername, message, force);
+ } catch (Exception e) {
+ LOG.warn("While stopping cluster " + e, e);
+ }
+ try {
+ sliderClient.actionDestroy(clustername);
+ } catch (Exception e) {
+ LOG.warn("While destroying cluster " + e, e);
+ }
}
}
return 0;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/YarnZKMiniClusterTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/YarnZKMiniClusterTestBase.java
index 322b346..3fe2942 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/YarnZKMiniClusterTestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/YarnZKMiniClusterTestBase.java
@@ -109,9 +109,7 @@ protected String createMiniCluster(String name,
int numLogDirs,
boolean startZK,
boolean startHDFS) throws IOException {
- if (SliderUtils.isUnset(name)) {
- name = methodName.getMethodName();
- }
+ name = buildClustername(name);
createMicroZKCluster("-" + name, conf);
conf.setBoolean(RegistryConstants.KEY_REGISTRY_ENABLED, true);
conf.set(RegistryConstants.KEY_REGISTRY_ZK_QUORUM, getZKBinding());
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app-override-resolved.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app-override-resolved.json
deleted file mode 100644
index e2a21ea..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app-override-resolved.json
+++ /dev/null
@@ -1,49 +0,0 @@
-{
- "name": "app-1",
- "lifetime": "3600",
- "configuration": {
- "properties": {
- "g1": "a",
- "g2": "b"
- }
- },
- "resource": {
- "cpus": 1,
- "memory": "512"
- },
- "number_of_containers": 2,
- "components": [
- {
- "name": "simple",
- "configuration": {
- "properties": {
- "g1": "a",
- "g2": "b"
- }
- }
- },
- {
- "name": "master",
- "configuration": {
- "properties": {
- "g1": "overridden",
- "g2": "b"
- }
- }
- },
- {
- "name": "worker",
- "resource": {
- "cpus": 1,
- "memory": "1024"
- },
- "configuration": {
- "properties": {
- "g1": "overridden-by-worker",
- "g2": "b",
- "timeout": "1000"
- }
- }
- }
- ]
-}
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app-override.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app-override.json
index 552cdef..19393bf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app-override.json
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app-override.json
@@ -5,7 +5,24 @@
"properties": {
"g1": "a",
"g2": "b"
- }
+ },
+ "files": [
+ {
+ "type": "PROPERTIES",
+ "dest_file": "file1",
+ "props": {
+ "k1": "v1",
+ "k2": "v2"
+ }
+ },
+ {
+ "type": "XML",
+ "dest_file": "file2",
+ "props": {
+ "k3": "v3"
+ }
+ }
+ ]
},
"resource": {
"cpus": 1,
@@ -14,7 +31,18 @@
"number_of_containers": 2,
"components": [
{
- "name": "simple"
+ "name": "simple",
+ "configuration": {
+ "files": [
+ {
+ "type": "PROPERTIES",
+ "dest_file": "file1",
+ "props": {
+ "k1": "overridden"
+ }
+ }
+ ]
+ }
},
{
"name": "master",
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app-resolved.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app-resolved.json
deleted file mode 100644
index cd1ab6f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app-resolved.json
+++ /dev/null
@@ -1,81 +0,0 @@
-{
- "name": "zk-app-1",
- "lifetime": "3600",
- "configuration": {
- "properties": {
- "internal.chaos.monkey.interval.seconds": "60",
- "zookeeper.port": "2181",
- "zookeeper.path": "/yarnapps_small_cluster",
- "zookeeper.hosts": "zoo1,zoo2,zoo3",
- "env.MALLOC_ARENA_MAX": "4",
- "site.hbase.master.startup.retainassign": "true",
- "site.fs.defaultFS": "hdfs://cluster:8020",
- "site.fs.default.name": "hdfs://cluster:8020",
- "site.hbase.master.info.port": "0",
- "site.hbase.regionserver.info.port": "0"
- }
- },
- "resource": {
- "cpus": 1,
- "memory": "512"
- },
- "number_of_containers": 2,
- "components": [
- {
- "name": "simple",
- "number_of_containers": 2,
- "configuration": {
- "properties": {
- "g1": "a",
- "g2": "b"
- }
- }
- },
- {
- "name": "master",
- "number_of_containers": 1,
- "resource": {
- "cpus": 1,
- "memory": "512"
- },
- "configuration": {
- "properties": {
- "zookeeper.port": "2181",
- "zookeeper.path": "/yarnapps_small_cluster",
- "zookeeper.hosts": "zoo1,zoo2,zoo3",
- "env.MALLOC_ARENA_MAX": "4",
- "site.hbase.master.startup.retainassign": "true",
- "site.fs.defaultFS": "hdfs://cluster:8020",
- "site.fs.default.name": "hdfs://cluster:8020",
- "site.hbase.master.info.port": "0",
- "site.hbase.regionserver.info.port": "0",
- "jvm.heapsize": "512M"
- }
- }
- },
- {
- "name": "worker",
- "number_of_containers": 5,
- "resource": {
- "cpus": 1,
- "memory": "1024"
- },
- "configuration": {
- "properties": {
- "g1": "overridden-by-worker",
- "g2": "b",
- "zookeeper.port": "2181",
- "zookeeper.path": "/yarnapps_small_cluster",
- "zookeeper.hosts": "zoo1,zoo2,zoo3",
- "env.MALLOC_ARENA_MAX": "4",
- "site.hbase.master.startup.retainassign": "true",
- "site.fs.defaultFS": "hdfs://cluster:8020",
- "site.fs.default.name": "hdfs://cluster:8020",
- "site.hbase.master.info.port": "0",
- "site.hbase.regionserver.info.port": "0",
- "jvm.heapsize": "512M"
- }
- }
- }
- ]
-}
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app.json
index 90857db..7a7810c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app.json
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app.json
@@ -5,16 +5,7 @@
"properties": {
"g1": "a",
"g2": "b",
- "internal.chaos.monkey.interval.seconds": "60",
- "zookeeper.port": "2181",
- "zookeeper.path": "/yarnapps_small_cluster",
- "zookeeper.hosts": "zoo1,zoo2,zoo3",
- "env.MALLOC_ARENA_MAX": "4",
- "site.hbase.master.startup.retainassign": "true",
- "site.fs.defaultFS": "hdfs://cluster:8020",
- "site.fs.default.name": "hdfs://cluster:8020",
- "site.hbase.master.info.port": "0",
- "site.hbase.regionserver.info.port": "0"
+ "internal.chaos.monkey.interval.seconds": "60"
}
},
"resource": {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/default.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/default.json
new file mode 100644
index 0000000..16f0efc
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/default.json
@@ -0,0 +1,16 @@
+{
+ "name": "default-app-1",
+ "lifetime": "3600",
+ "components" :
+ [
+ {
+ "name": "SLEEP",
+ "number_of_containers": 1,
+ "launch_command": "sleep 3600",
+ "resource": {
+ "cpus": 2,
+ "memory": "256"
+ }
+ }
+ ]
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/external0.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/external0.json
new file mode 100644
index 0000000..1f9dfeb
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/external0.json
@@ -0,0 +1,8 @@
+{
+ "name": "external-0",
+ "lifetime": "3600",
+ "artifact": {
+ "type": "APPLICATION",
+ "id": "app-1"
+ }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/external1.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/external1.json
new file mode 100644
index 0000000..d7bf9bb
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/external1.json
@@ -0,0 +1,21 @@
+{
+ "name": "external-1",
+ "lifetime": "3600",
+ "components": [
+ {
+ "name": "simple",
+ "artifact": {
+ "type": "APPLICATION",
+ "id": "app-1"
+ }
+ },
+ {
+ "name": "other",
+ "number_of_containers": 2,
+ "resource": {
+ "cpus": 1,
+ "memory": "512"
+ }
+ }
+ ]
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/external2.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/external2.json
new file mode 100644
index 0000000..c4337a1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/external2.json
@@ -0,0 +1,21 @@
+{
+ "name": "external-2",
+ "lifetime": "3600",
+ "components": [
+ {
+ "name": "ext",
+ "artifact": {
+ "type": "APPLICATION",
+ "id": "external-1"
+ }
+ },
+ {
+ "name": "another",
+ "number_of_containers": 1,
+ "resource": {
+ "cpus": 1,
+ "memory": "512"
+ }
+ }
+ ]
+}