diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/services/api/impl/TestApplicationApiService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/services/api/impl/TestApplicationApiService.java index 6e077d2..1657249 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/services/api/impl/TestApplicationApiService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/services/api/impl/TestApplicationApiService.java @@ -77,7 +77,8 @@ public void testValidateApplicationPostPayload() throws Exception { ServiceApiUtil.validateApplicationPayload(app, null); Assert.fail(EXCEPTION_PREFIX + "application with bad name " + badName); } catch (IllegalArgumentException e) { - Assert.assertEquals(ERROR_APPLICATION_NAME_INVALID_FORMAT, + Assert.assertEquals(String.format( + ERROR_APPLICATION_NAME_INVALID_FORMAT, badName), e.getMessage()); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Component.java index e7f3796..f73c305 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Component.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Component.java @@ -50,9 +50,9 @@ private String name = null; private List dependencies = new ArrayList(); private ReadinessCheck readinessCheck = null; - private Artifact artifact = new Artifact(); + private Artifact artifact = null; private String launchCommand = null; - private Resource resource = new Resource(); + private Resource resource = null; private Long numberOfContainers = null; private Boolean uniqueComponentSupport = false; private Boolean runPrivilegedContainer = false; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java index 32d78b4..22bf33c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -101,6 +101,7 @@ import org.apache.slider.common.params.ClientArgs; import org.apache.slider.common.params.CommonArgs; import org.apache.slider.common.tools.ConfigHelper; +import org.apache.slider.common.tools.Duration; import org.apache.slider.common.tools.SliderFileSystem; import org.apache.slider.common.tools.SliderUtils; import org.apache.slider.common.tools.SliderVersionInfo; @@ -636,16 +637,16 @@ private Application getApplicationFromArgs(String clusterName, public int actionBuild(Application application) throws YarnException, IOException { Path appDir = checkAppNotExistOnHdfs(application); + ServiceApiUtil.validateApplicationPayload(application, sliderFileSystem); persistApp(appDir, application); return EXIT_SUCCESS; } public ApplicationId actionCreate(Application application) throws IOException, YarnException { - ServiceApiUtil.validateApplicationPayload(application, - sliderFileSystem.getFileSystem()); String appName = application.getName(); validateClusterName(appName); + ServiceApiUtil.validateApplicationPayload(application, sliderFileSystem); verifyNoLiveApp(appName, "Create"); Path appDir = checkAppNotExistOnHdfs(application); @@ -880,6 +881,14 @@ private Path checkAppNotExistOnHdfs(Application application) return appDir; } + private Path checkAppExistOnHdfs(String appName) + throws IOException, SliderException { + Path appDir = sliderFileSystem.buildClusterDirPath(appName); + sliderFileSystem.verifyPathExists( + new Path(appDir, appName + ".json")); + return appDir; + } + private void persistApp(Path appDir, Application application) throws IOException, SliderException { FsPermission appDirPermission = new FsPermission("750"); @@ -1802,23 +1811,24 @@ public int actionStop(String appName, ActionFreezeArgs freezeArgs) public int actionStart(String appName, ActionThawArgs thaw) throws YarnException, IOException { validateClusterName(appName); + Path appDir = checkAppExistOnHdfs(appName); + Application application = ServiceApiUtil.loadApplication(sliderFileSystem, + appName); + ServiceApiUtil.validateApplicationPayload(application, sliderFileSystem); // see if it is actually running and bail out; verifyNoLiveApp(appName, "Thaw"); - Path appDir = sliderFileSystem.buildClusterDirPath(appName); - Path appJson = new Path(appDir, appName + ".json"); - Application application = - jsonSerDeser.load(sliderFileSystem.getFileSystem(), appJson); - submitApp(application); + ApplicationId appId = submitApp(application); + application.setId(appId.toString()); + // write app definition on to hdfs + persistApp(appDir, application); return 0; } public Map flex(String appName, Map componentCounts) throws YarnException, IOException { validateClusterName(appName); - Path appDir = sliderFileSystem.buildClusterDirPath(appName); - Path appJson = new Path(appDir, appName + ".json"); - Application persistedApp = - jsonSerDeser.load(sliderFileSystem.getFileSystem(), appJson); + Application persistedApp = ServiceApiUtil.loadApplication(sliderFileSystem, + appName); Map original = new HashMap<>(componentCounts.size()); for (Component persistedComp : persistedApp.getComponents()) { String name = persistedComp.getName(); @@ -1833,7 +1843,8 @@ public int actionStart(String appName, ActionThawArgs thaw) + " do not exist in app definition."); } jsonSerDeser - .save(sliderFileSystem.getFileSystem(), appJson, persistedApp, true); + .save(sliderFileSystem.getFileSystem(), ServiceApiUtil.getAppJsonPath( + sliderFileSystem, appName), persistedApp, true); log.info("Updated app definition file for components " + componentCounts .keySet()); @@ -2705,6 +2716,18 @@ private int actionTokens(ActionTokensArgs args) yarnClient); } + @VisibleForTesting + public ApplicationReport monitorAppToRunning(Duration duration) + throws YarnException, IOException { + return monitorAppToState(YarnApplicationState.RUNNING, duration); + } + + @VisibleForTesting + public ApplicationReport monitorAppToState(YarnApplicationState desiredState, + Duration duration) + throws YarnException, IOException { + return yarnClient.monitorAppToState(applicationId, desiredState, duration); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java index 4839395..6f24691 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java @@ -245,4 +245,64 @@ public NodeInformationList listNodes(String label, boolean live) } return results; } + + /** + * Monitor the submitted application for reaching the requested state. + * Will also report if the app reaches a later state (failed, killed, etc) + * Kill application if duration!= null & time expires. + * @param appId Application Id of application to be monitored + * @param duration how long to wait -must be more than 0 + * @param desiredState desired state. + * @return the application report -null on a timeout + * @throws YarnException + * @throws IOException + */ + public ApplicationReport monitorAppToState( + ApplicationId appId, YarnApplicationState desiredState, Duration duration) + throws YarnException, IOException { + + if (appId == null) { + throw new BadCommandArgumentsException("null application ID"); + } + if (duration.limit <= 0) { + throw new BadCommandArgumentsException("Invalid monitoring duration"); + } + log.debug("Waiting {} millis for app to reach state {} ", + duration.limit, + desiredState); + duration.start(); + try { + while (true) { + // Get application report for the appId we are interested in + + ApplicationReport r = getApplicationReport(appId); + + log.debug("queried status is\n{}", + new SliderUtils.OnDemandReportStringifier(r)); + + YarnApplicationState state = r.getYarnApplicationState(); + if (state.ordinal() >= desiredState.ordinal()) { + log.debug("App in desired state (or higher) :{}", state); + return r; + } + if (duration.getLimitExceeded()) { + log.debug( + "Wait limit of {} millis to get to state {}, exceeded; app status\n {}", + duration.limit, + desiredState, + new SliderUtils.OnDemandReportStringifier(r)); + return null; + } + + // sleep 1s. + try { + Thread.sleep(1000); + } catch (InterruptedException ignored) { + log.debug("Thread sleep in monitoring loop interrupted"); + } + } + } finally { + duration.close(); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java index bc8e139..9f00c47 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java @@ -2540,14 +2540,4 @@ public static long getTimeRange(org.apache.slider.api.resource long totalMinutes = days * 24 * 60 + hours * 24 + minutes; return totalMinutes * 60 + seconds; } - - public static void resolve(Application application) { - org.apache.slider.api.resource.Configuration global = application - .getConfiguration(); - for (Component component : application.getComponents()) { - mergeMapsIgnoreDuplicateKeys(component.getConfiguration().getProperties(), - global.getProperties()); - } - // TODO merge other information to components - } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java index 84dde08..d5b533a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java @@ -107,7 +107,6 @@ import org.apache.slider.core.main.LauncherExitCodes; import org.apache.slider.core.main.RunService; import org.apache.slider.core.main.ServiceLauncher; -import org.apache.slider.core.persist.JsonSerDeser; import org.apache.slider.core.registry.info.CustomRegistryConstants; import org.apache.slider.providers.ProviderCompleted; import org.apache.slider.providers.ProviderService; @@ -157,6 +156,7 @@ import org.apache.slider.server.services.workflow.WorkflowExecutorService; import org.apache.slider.server.services.workflow.WorkflowRpcService; import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders; +import org.apache.slider.util.ServiceApiUtil; import org.codehaus.jackson.map.PropertyNamingStrategy; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -389,9 +389,6 @@ */ private boolean securityEnabled; private ContentCache contentCache; - private static final JsonSerDeser jsonSerDeser = - new JsonSerDeser(Application.class, - PropertyNamingStrategy.CAMEL_CASE_TO_LOWER_CASE_WITH_UNDERSCORES); /** * resource limits @@ -590,9 +587,7 @@ private int createAndRunCluster(String appName) throws Throwable { Path appDir = new Path((serviceArgs.getAppDefDir())); SliderFileSystem fs = getClusterFS(); fs.setAppDir(appDir); - Path appJson = new Path(appDir, appName + ".json"); - log.info("Loading application definition from " + appJson); - application = jsonSerDeser.load(fs.getFileSystem(), appJson); + application = ServiceApiUtil.loadApplication(fs, appName); log.info("Application Json: " + application); stateForProviders.setApplicationName(appName); Configuration serviceConf = getConfig(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java index ac89ed8..f72afd8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java @@ -21,7 +21,7 @@ String ERROR_APPLICATION_NAME_INVALID = "Application name is either empty or not provided"; String ERROR_APPLICATION_NAME_INVALID_FORMAT = - "Application name is not valid - only lower case letters, digits," + "Application name %s is not valid - only lower case letters, digits," + " underscore and hyphen are allowed"; String ERROR_APPLICATION_NOT_RUNNING = "Application not running"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/ServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/ServiceApiUtil.java index d7c72a3..1758abf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/ServiceApiUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/ServiceApiUtil.java @@ -28,12 +28,17 @@ import org.apache.slider.api.resource.ConfigFile; import org.apache.slider.api.resource.Configuration; import org.apache.slider.api.resource.Resource; +import org.apache.slider.common.tools.SliderFileSystem; import org.apache.slider.common.tools.SliderUtils; +import org.apache.slider.core.persist.JsonSerDeser; +import org.apache.slider.providers.AbstractClientProvider; +import org.apache.slider.providers.SliderProviderFactory; +import org.codehaus.jackson.map.PropertyNamingStrategy; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; -import java.nio.file.Paths; +import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -41,102 +46,141 @@ public class ServiceApiUtil { private static final Logger log = LoggerFactory.getLogger(ServiceApiUtil.class); + private static final JsonSerDeser jsonSerDeser = + new JsonSerDeser<>(Application.class, + PropertyNamingStrategy.CAMEL_CASE_TO_LOWER_CASE_WITH_UNDERSCORES); + @VisibleForTesting public static void validateApplicationPayload(Application application, - FileSystem fs) throws IOException { + SliderFileSystem fs) throws IOException { if (StringUtils.isEmpty(application.getName())) { throw new IllegalArgumentException( RestApiErrorMessages.ERROR_APPLICATION_NAME_INVALID); } if (!SliderUtils.isClusternameValid(application.getName())) { - throw new IllegalArgumentException( - RestApiErrorMessages.ERROR_APPLICATION_NAME_INVALID_FORMAT); + throw new IllegalArgumentException(String.format( + RestApiErrorMessages.ERROR_APPLICATION_NAME_INVALID_FORMAT, + application.getName())); } // If the application has no components do top-level checks if (!hasComponent(application)) { - // artifact - if (application.getArtifact() == null) { - throw new IllegalArgumentException( - RestApiErrorMessages.ERROR_ARTIFACT_INVALID); - } - if (StringUtils.isEmpty(application.getArtifact().getId())) { - throw new IllegalArgumentException( - RestApiErrorMessages.ERROR_ARTIFACT_ID_INVALID); - } - // If artifact is of type APPLICATION, add a slider specific property - if (application.getArtifact().getType() - == Artifact.TypeEnum.APPLICATION) { - if (application.getConfiguration() == null) { - application.setConfiguration(new Configuration()); + if (application.getArtifact() != null && application.getArtifact() + .getType() == Artifact.TypeEnum.APPLICATION) { + if (StringUtils.isEmpty(application.getArtifact().getId())) { + throw new IllegalArgumentException( + RestApiErrorMessages.ERROR_ARTIFACT_ID_INVALID); } - } - // resource - validateApplicationResource(application.getResource(), null, - application.getArtifact().getType()); - - // container size - if (application.getNumberOfContainers() == null - || application.getNumberOfContainers() < 0) { - throw new IllegalArgumentException( - RestApiErrorMessages.ERROR_CONTAINERS_COUNT_INVALID + ": " - + application.getNumberOfContainers()); - } - validateConfigFile(application.getConfiguration().getFiles(), fs); - // Since it is a simple app with no components, create a default component - application.getComponents().add(createDefaultComponent(application)); - } else { - // If the application has components, then run checks for each component. - // Let global values take effect if component level values are not - // provided. - Artifact globalArtifact = application.getArtifact(); - Resource globalResource = application.getResource(); - Long globalNumberOfContainers = application.getNumberOfContainers(); - for (Component comp : application.getComponents()) { + List applicationComponents = getApplicationComponents(fs, + application.getArtifact().getId()); + application.setComponents(applicationComponents); + application.setArtifact(null); + } else { // artifact - if (comp.getArtifact() == null) { - comp.setArtifact(globalArtifact); - } - // If still null raise validation exception - if (comp.getArtifact() == null) { - throw new IllegalArgumentException(String - .format(RestApiErrorMessages.ERROR_ARTIFACT_FOR_COMP_INVALID, - comp.getName())); + AbstractClientProvider clientProvider = SliderProviderFactory + .getClientProvider(application.getArtifact()); + clientProvider.validateArtifact(application.getArtifact(), fs + .getFileSystem()); + + // resource + validateApplicationResource(application.getResource(), null); + + // container size + if (application.getNumberOfContainers() == null + || application.getNumberOfContainers() < 0) { + throw new IllegalArgumentException( + RestApiErrorMessages.ERROR_CONTAINERS_COUNT_INVALID + ": " + + application.getNumberOfContainers()); } + validateConfigFile(application.getConfiguration().getFiles(), fs + .getFileSystem()); + clientProvider.validateConfigFiles(application.getConfiguration() + .getFiles(), fs.getFileSystem()); + // Since it is a simple app with no components, create a default + // component + application.getComponents().add(createDefaultComponent(application)); + } + } + // Now run checks for each component. + // Let global values take effect if component level values are not + // provided. + Artifact globalArtifact = application.getArtifact(); + Resource globalResource = application.getResource(); + Configuration globalConf = application.getConfiguration(); + Long globalNumberOfContainers = application.getNumberOfContainers(); + Set componentNames = new HashSet<>(); + List componentsToRemove = new ArrayList<>(); + List componentsToAdd = new ArrayList<>(); + for (Component comp : application.getComponents()) { + if (componentNames.contains(comp.getName())) { + throw new IllegalArgumentException("Component name collision: " + + comp.getName()); + } + componentNames.add(comp.getName()); + // artifact + if (comp.getArtifact() == null) { + comp.setArtifact(globalArtifact); + } + //configuration + SliderUtils.mergeMapsIgnoreDuplicateKeys(comp.getConfiguration() + .getProperties(), globalConf.getProperties()); + SliderUtils.mergeMapsIgnoreDuplicateKeys(comp.getConfiguration() + .getEnv(), globalConf.getEnv()); + if (comp.getConfiguration().getFiles().isEmpty()) { + comp.getConfiguration().setFiles(globalConf.getFiles()); + } + // If artifact is of type APPLICATION, add a slider specific property + if (comp.getArtifact() != null && comp.getArtifact().getType() == + Artifact.TypeEnum.APPLICATION) { if (StringUtils.isEmpty(comp.getArtifact().getId())) { - throw new IllegalArgumentException(String - .format(RestApiErrorMessages.ERROR_ARTIFACT_ID_FOR_COMP_INVALID, - comp.getName())); + throw new IllegalArgumentException( + RestApiErrorMessages.ERROR_ARTIFACT_ID_INVALID); } - - // If artifact is of type APPLICATION, add a slider specific property - if (comp.getArtifact().getType() == Artifact.TypeEnum.APPLICATION) { - if (comp.getConfiguration() == null) { - comp.setConfiguration(new Configuration()); + componentsToRemove.add(comp); + List applicationComponents = getApplicationComponents(fs, + comp.getArtifact().getId()); + for (Component c : applicationComponents) { + if (componentNames.contains(c.getName())) { + // TODO allow name collisions? see AppState#roles + // TODO or add prefix to external component names? + throw new IllegalArgumentException("Component name collision: " + + c.getName()); } - comp.setName(comp.getArtifact().getId()); + componentNames.add(c.getName()); } + componentsToAdd.addAll(applicationComponents); + } + } + application.getComponents().removeAll(componentsToRemove); + application.getComponents().addAll(componentsToAdd); - // resource - if (comp.getResource() == null) { - comp.setResource(globalResource); - } - validateApplicationResource(comp.getResource(), comp, - comp.getArtifact().getType()); + for (Component comp : application.getComponents()) { + AbstractClientProvider compClientProvider = SliderProviderFactory + .getClientProvider(comp.getArtifact()); + compClientProvider.validateArtifact(comp.getArtifact(), fs + .getFileSystem()); - // container count - if (comp.getNumberOfContainers() == null) { - comp.setNumberOfContainers(globalNumberOfContainers); - } - if (comp.getNumberOfContainers() == null - || comp.getNumberOfContainers() < 0) { - throw new IllegalArgumentException(String.format( - RestApiErrorMessages.ERROR_CONTAINERS_COUNT_FOR_COMP_INVALID - + ": " + comp.getNumberOfContainers(), comp.getName())); - } - validateConfigFile(comp.getConfiguration().getFiles(), fs); + // resource + if (comp.getResource() == null) { + comp.setResource(globalResource); + } + validateApplicationResource(comp.getResource(), comp); + + // container count + if (comp.getNumberOfContainers() == null) { + comp.setNumberOfContainers(globalNumberOfContainers); } + if (comp.getNumberOfContainers() == null + || comp.getNumberOfContainers() < 0) { + throw new IllegalArgumentException(String.format( + RestApiErrorMessages.ERROR_CONTAINERS_COUNT_FOR_COMP_INVALID + + ": " + comp.getNumberOfContainers(), comp.getName())); + } + validateConfigFile(comp.getConfiguration().getFiles(), fs + .getFileSystem()); + compClientProvider.validateConfigFiles(comp.getConfiguration() + .getFiles(), fs.getFileSystem()); } // Application lifetime if not specified, is set to unlimited lifetime @@ -145,8 +189,29 @@ public static void validateApplicationPayload(Application application, } } + @VisibleForTesting + public static List getApplicationComponents(SliderFileSystem + fs, String appName) throws IOException { + return loadApplication(fs, appName).getComponents(); + } + + + public static Application loadApplication(SliderFileSystem fs, String + appName) throws IOException { + Path appJson = getAppJsonPath(fs, appName); + log.info("Loading application definition from " + appJson); + Application externalApplication = jsonSerDeser.load(fs + .getFileSystem(), appJson); + return externalApplication; + } + + public static Path getAppJsonPath(SliderFileSystem fs, String appName) { + Path appDir = fs.buildClusterDirPath(appName); + Path appJson = new Path(appDir, appName + ".json"); + return appJson; + } + // 1) Verify the src_file exists and non-empty for template - // 2) dest_file is absolute path private static void validateConfigFile(List list, FileSystem fs) throws IOException { Set destFileSet = new HashSet<>(); @@ -170,11 +235,6 @@ private static void validateConfigFile(List list, FileSystem fs) if (StringUtils.isEmpty(file.getDestFile())) { throw new IllegalArgumentException("Dest_file is empty."); } - // validate dest_file is absolute - if (!Paths.get(file.getDestFile()).isAbsolute()) { - throw new IllegalArgumentException( - "Dest_file must be absolute path: " + file.getDestFile()); - } if (destFileSet.contains(file.getDestFile())) { throw new IllegalArgumentException( @@ -186,11 +246,8 @@ private static void validateConfigFile(List list, FileSystem fs) private static void validateApplicationResource(Resource resource, - Component comp, Artifact.TypeEnum artifactType) { + Component comp) { // Only apps/components of type APPLICATION can skip resource requirement - if (resource == null && artifactType == Artifact.TypeEnum.APPLICATION) { - return; - } if (resource == null) { throw new IllegalArgumentException( comp == null ? RestApiErrorMessages.ERROR_RESOURCE_INVALID : String diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/ExampleConfResources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/ExampleConfResources.java index f13fbcc..a27fd1a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/ExampleConfResources.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/ExampleConfResources.java @@ -32,15 +32,16 @@ public final class ExampleConfResources { public static final String APP_JSON = "app.json"; - public static final String APP_RES = "app-resolved.json"; public static final String OVERRIDE_JSON = "app-override.json"; - public static final String OVERRIDE_RES = "app-override-resolved.json"; + public static final String DEFAULT_JSON = "default.json"; + public static final String EXTERNAL_JSON_1 = "external1.json"; + public static final String EXTERNAL_JSON_2 = "external2.json"; public static final String PACKAGE = "/org/apache/slider/core/conf/examples/"; - private static final String[] ALL_EXAMPLES = {APP_JSON, APP_RES, - OVERRIDE_JSON, OVERRIDE_RES}; + private static final String[] ALL_EXAMPLES = {APP_JSON, OVERRIDE_JSON, + DEFAULT_JSON, EXTERNAL_JSON_1, EXTERNAL_JSON_2}; public static final List ALL_EXAMPLE_RESOURCES = new ArrayList<>(); static { @@ -55,4 +56,8 @@ private ExampleConfResources() { static Application loadResource(String name) throws IOException { return JSON_SER_DESER.fromResource(PACKAGE + name); } + + public static String resourceName(String name) { + return "target/test-classes" + PACKAGE + name; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestConfTreeLoadExamples.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestConfTreeLoadExamples.java index 48b0736..5d4e09b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestConfTreeLoadExamples.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestConfTreeLoadExamples.java @@ -18,13 +18,18 @@ package org.apache.slider.core.conf; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.slider.api.resource.Application; +import org.apache.slider.common.tools.SliderFileSystem; import org.apache.slider.common.tools.SliderUtils; +import org.apache.slider.util.ServiceApiUtil; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; +import java.net.URI; import java.util.Arrays; import java.util.Collection; @@ -56,7 +61,11 @@ public TestConfTreeLoadExamples(String resource) { public void testLoadResource() throws Throwable { try { Application application = JSON_SER_DESER.fromResource(resource); - SliderUtils.resolve(application); + YarnConfiguration conf = SliderUtils.createConfiguration(); + FileSystem fs = FileSystem.get(new URI("file:///"), conf); + SliderFileSystem sliderFileSystem = new SliderFileSystem(fs, conf); + + ServiceApiUtil.validateApplicationPayload(application, sliderFileSystem); } catch (Exception e) { throw new Exception("exception loading " + resource + ":" + e.toString()); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestConfigurationResolve.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestConfigurationResolve.java index 285ddfa..8781881 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestConfigurationResolve.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestConfigurationResolve.java @@ -18,14 +18,20 @@ package org.apache.slider.core.conf; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.slider.api.resource.Application; import org.apache.slider.api.resource.Configuration; +import org.apache.slider.common.tools.SliderFileSystem; import org.apache.slider.common.tools.SliderUtils; +import org.apache.slider.util.ServiceApiUtil; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.net.URI; + import static org.apache.slider.api.InternalKeys.CHAOS_MONKEY_INTERVAL; import static org.apache.slider.api.InternalKeys.DEFAULT_CHAOS_MONKEY_INTERVAL_DAYS; import static org.apache.slider.api.InternalKeys.DEFAULT_CHAOS_MONKEY_INTERVAL_HOURS; @@ -66,7 +72,10 @@ public void testOverride() throws Throwable { assertEquals("1000", worker.getProperty("timeout")); // here is the resolution - SliderUtils.resolve(orig); + YarnConfiguration conf = SliderUtils.createConfiguration(); + FileSystem fs = FileSystem.get(new URI("file:///"), conf); + SliderFileSystem sliderFileSystem = new SliderFileSystem(fs, conf); + ServiceApiUtil.validateApplicationPayload(orig, sliderFileSystem); global = orig.getConfiguration(); LOG.info("global = {}", global); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestBuildApplicationComponent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestBuildApplicationComponent.java new file mode 100644 index 0000000..46e6d38 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestBuildApplicationComponent.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.providers; + +import org.apache.slider.api.resource.Component; +import org.apache.slider.client.SliderClient; +import org.apache.slider.common.params.SliderActions; +import org.apache.slider.common.tools.SliderFileSystem; +import org.apache.slider.core.conf.ExampleConfResources; +import org.apache.slider.core.main.ServiceLauncher; +import org.apache.slider.util.ServiceApiUtil; +import org.apache.slider.utils.YarnZKMiniClusterTestBase; +import org.junit.Test; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static org.apache.slider.common.params.Arguments.ARG_APPDEF; + +public class TestBuildApplicationComponent extends YarnZKMiniClusterTestBase { + + private static void checkComponentNames(List components, + Set names) { + assertEquals(names.size(), components.size()); + for (Component comp : components) { + assertTrue(names.contains(comp.getName())); + } + } + + @Test + public void testExternalComponentBuild() throws Throwable { + String clustername = createMiniCluster("", getConfiguration(), 1, true); + + describe("verify external components"); + + String appDef0 = ExampleConfResources.resourceName(ExampleConfResources + .APP_JSON); + String appDef1 = ExampleConfResources.resourceName(ExampleConfResources + .EXTERNAL_JSON_1); + String appDef2 = ExampleConfResources.resourceName(ExampleConfResources + .EXTERNAL_JSON_2); + + ServiceLauncher launcher = createOrBuildCluster( + SliderActions.ACTION_BUILD, "app-1", Arrays.asList(ARG_APPDEF, + appDef0), true, false); + SliderClient sliderClient = launcher.getService(); + addToTeardown(sliderClient); + + // verify the cluster exists + assertEquals(0, sliderClient.actionExists(clustername, false)); + + String parent1 = "external-1"; + launcher = createOrBuildCluster(SliderActions.ACTION_BUILD, parent1, + Arrays.asList(ARG_APPDEF, appDef1), true, false); + sliderClient = launcher.getService(); + addToTeardown(sliderClient); + + // verify the cluster exists + assertEquals(0, sliderClient.actionExists(parent1, false)); + // verify generated conf + SliderFileSystem sfs = createSliderFileSystem(); + List components = ServiceApiUtil.getApplicationComponents(sfs, + parent1); + Set nameSet = new HashSet<>(); + nameSet.add("simple"); + nameSet.add("master"); + nameSet.add("worker"); + nameSet.add("other"); + checkComponentNames(components, nameSet); + + String parent2 = "external-2"; + launcher = createOrBuildCluster(SliderActions.ACTION_BUILD, parent2, + Arrays.asList(ARG_APPDEF, appDef2), true, false); + sliderClient = launcher.getService(); + addToTeardown(sliderClient); + + // verify the cluster exists + assertEquals(0, sliderClient.actionExists(parent2, false)); + // verify generated conf + components = ServiceApiUtil.getApplicationComponents(sfs, parent2); + nameSet.add("another"); + checkComponentNames(components, nameSet); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestDefaultProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestDefaultProvider.java new file mode 100644 index 0000000..2b59de9 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestDefaultProvider.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.providers; + +import org.apache.slider.api.resource.Application; +import org.apache.slider.client.SliderClient; +import org.apache.slider.common.params.SliderActions; +import org.apache.slider.core.conf.ExampleConfResources; +import org.apache.slider.core.main.ServiceLauncher; +import org.apache.slider.utils.YarnZKMiniClusterTestBase; +import org.junit.Ignore; +import org.junit.Test; + +import java.util.Arrays; + +import static org.apache.slider.common.params.Arguments.ARG_APPDEF; + +public class TestDefaultProvider extends YarnZKMiniClusterTestBase { + + // TODO figure out how to run client commands against minicluster + // (currently errors out unable to find containing jar of AM for upload) + @Ignore + @Test + public void testDefaultProvider() throws Throwable { + createMiniCluster("", getConfiguration(), 1, true); + String appName = "default-1"; + + describe("verify default provider"); + + String appDef = ExampleConfResources.resourceName(ExampleConfResources + .DEFAULT_JSON); + + ServiceLauncher launcher = createOrBuildCluster( + SliderActions.ACTION_CREATE, appName, Arrays.asList(ARG_APPDEF, + appDef), true, true); + SliderClient sliderClient = launcher.getService(); + addToTeardown(sliderClient); + + Application application = sliderClient.actionStatus(appName); + assertEquals(1L, application.getContainers().size()); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/BaseMockAppStateAATest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/BaseMockAppStateAATest.java index c1f2886..6f4ca42 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/BaseMockAppStateAATest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/BaseMockAppStateAATest.java @@ -43,7 +43,7 @@ @Override public Application buildApplication() { Application application = factory.newApplication(0, 0, 0) - .name(getTestName()); + .name(getValidTestName()); application.getComponent(ROLE1).getConfiguration().setProperty( COMPONENT_PLACEMENT_POLICY, Integer.toString(PlacementPolicy .ANTI_AFFINITY_REQUIRED)); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.java index eb25b40..571e9d9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.java @@ -362,7 +362,7 @@ public void testAMRestart() throws Throwable { // now destroy the app state AppStateBindingInfo bindingInfo = buildBindingInfo(); bindingInfo.application = factory.newApplication(0, 0, desiredAA).name( - getTestName()); + getValidTestName()); bindingInfo.application.getComponent(ROLE2) .getConfiguration().setProperty(COMPONENT_PLACEMENT_POLICY, Integer.toString(PlacementPolicy.ANTI_AFFINITY_REQUIRED)); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.java index ea0dcf4..9cbda4f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.java @@ -203,7 +203,7 @@ public void testRecurrentStartupFailureWithUnlimitedFailures() throws // Update instance definition to allow containers to fail any number of // times AppStateBindingInfo bindingInfo = buildBindingInfo(); - bindingInfo.application.getConfiguration().setProperty( + bindingInfo.application.getComponent(ROLE0).getConfiguration().setProperty( ResourceKeys.CONTAINER_FAILURE_THRESHOLD, "0"); appState = new MockAppState(bindingInfo); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.java index 6d8e963..7f7f93a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.java @@ -38,6 +38,7 @@ import org.slf4j.LoggerFactory; import java.io.File; +import java.io.IOException; import java.util.Collections; /** @@ -65,7 +66,7 @@ public MockYarnEngine createYarnEngine() { } @Override - public AppStateBindingInfo buildBindingInfo() { + public AppStateBindingInfo buildBindingInfo() throws IOException { AppStateBindingInfo bindingInfo = super.buildBindingInfo(); bindingInfo.releaseSelector = new MostRecentContainerReleaseSelector(); return bindingInfo; @@ -145,7 +146,7 @@ public void testHistoryFlexSaveResetLoad() throws Throwable { appState = new MockAppState(); AppStateBindingInfo binding2 = buildBindingInfo(); binding2.application = factory.newApplication(0, 0, 0) - .name(getTestName()); + .name(getValidTestName()); binding2.historyPath = historyPath2; appState.buildInstance(binding2); // on this read there won't be the right number of roles diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.java index b0634bf..d9c675d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.java @@ -70,7 +70,7 @@ public void testRebuild() throws Throwable { AppStateBindingInfo bindingInfo = buildBindingInfo(); bindingInfo.application = factory.newApplication(r0, r1, r2) - .name(getTestName()); + .name(getValidTestName()); bindingInfo.liveContainers = containers; appState = new MockAppState(bindingInfo); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java index b7e967f..703d65f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java @@ -30,6 +30,7 @@ import org.apache.slider.server.appmaster.state.RoleStatus; import org.junit.Test; +import java.io.IOException; import java.util.Collections; import java.util.List; import java.util.Map; @@ -59,7 +60,7 @@ public MockYarnEngine createYarnEngine() { } @Override - public AppStateBindingInfo buildBindingInfo() { + public AppStateBindingInfo buildBindingInfo() throws IOException { AppStateBindingInfo bindingInfo = super.buildBindingInfo(); bindingInfo.releaseSelector = new MostRecentContainerReleaseSelector(); return bindingInfo; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.java index d382c8a..4aa5895 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.java @@ -40,7 +40,7 @@ @Override public Application buildApplication() { - return factory.newApplication(1, 0, 0).name(getTestName()); + return factory.newApplication(1, 0, 0).name(getValidTestName()); } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java index 69abccf..6a2bd2d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java @@ -51,6 +51,7 @@ import org.apache.slider.server.appmaster.state.RoleInstance; import org.apache.slider.server.appmaster.state.RoleStatus; import org.apache.slider.server.appmaster.state.StateAccessForProviders; +import org.apache.slider.util.ServiceApiUtil; import org.apache.slider.utils.SliderTestBase; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -62,6 +63,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Locale; import java.util.Map.Entry; /** @@ -118,7 +120,7 @@ protected void initApp() historyPath = new Path(historyWorkDir.toURI()); fs.delete(historyPath, true); appState = new MockAppState(buildBindingInfo()); - stateAccess = new ProviderAppState(getTestName(), appState); + stateAccess = new ProviderAppState(getValidTestName(), appState); } /** @@ -127,9 +129,11 @@ protected void initApp() * from {@link #buildApplication()} ()} * @return */ - protected AppStateBindingInfo buildBindingInfo() { + protected AppStateBindingInfo buildBindingInfo() throws IOException { AppStateBindingInfo binding = new AppStateBindingInfo(); binding.application = buildApplication(); + ServiceApiUtil.validateApplicationPayload(binding.application, + sliderFileSystem); //binding.roles = new ArrayList<>(factory.ROLES); binding.fs = fs; binding.historyPath = historyPath; @@ -142,7 +146,7 @@ protected AppStateBindingInfo buildBindingInfo() { * @return the instance definition */ public Application buildApplication() { - return factory.newApplication(0, 0, 0).name(getTestName()); + return factory.newApplication(0, 0, 0).name(getValidTestName()); } /** @@ -153,6 +157,10 @@ public String getTestName() { return methodName.getMethodName(); } + public String getValidTestName() { + return getTestName().toLowerCase(Locale.ENGLISH); + } + public RoleStatus getRole0Status() { return lookupRole(ROLE0); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockFactory.java index 2ac5087..d27cd33 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockFactory.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockFactory.java @@ -32,6 +32,7 @@ import org.apache.slider.api.ResourceKeys; import org.apache.slider.api.resource.Application; import org.apache.slider.api.resource.Component; +import org.apache.slider.api.resource.Resource; import org.apache.slider.providers.PlacementPolicy; import org.apache.slider.providers.ProviderRole; @@ -190,6 +191,7 @@ public MockContainer newContainer(AMRMClient.ContainerRequest req, String */ public Application newApplication(long r1, long r2, long r3) { Application application = new Application(); + application.setResource(new Resource().memory("256")); application.getConfiguration().setProperty(ResourceKeys .NODE_FAILURE_THRESHOLD, Integer.toString(NODE_FAILURE_THRESHOLD)); List components = application.getComponents(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/YarnMiniClusterTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/YarnMiniClusterTestBase.java index 746a0ec..279a9c5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/YarnMiniClusterTestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/YarnMiniClusterTestBase.java @@ -37,6 +37,7 @@ import org.apache.slider.common.SliderXmlConfKeys; import org.apache.slider.common.params.ActionFreezeArgs; import org.apache.slider.common.params.Arguments; +import org.apache.slider.common.params.SliderActions; import org.apache.slider.common.tools.Duration; import org.apache.slider.common.tools.SliderFileSystem; import org.apache.slider.common.tools.SliderUtils; @@ -502,6 +503,62 @@ protected String getFsDefaultName() { } /** + * Create or build a cluster (the action is set by the first verb) + * @param action operation to invoke: SliderActions.ACTION_CREATE or + * SliderActions.ACTION_BUILD + * @param clustername cluster name + * @param extraArgs list of extra args to add to the creation command + * @param deleteExistingData should the data of any existing cluster + * of this name be deleted + * @param blockUntilRunning block until the AM is running + * @return launcher which will have executed the command. + */ + public ServiceLauncher createOrBuildCluster(String action, + String clustername, List extraArgs, boolean deleteExistingData, + boolean blockUntilRunning) throws Throwable { + assertNotNull(clustername); + assertNotNull(miniCluster); + // update action should keep existing data + Configuration config = miniCluster.getConfig(); + if (deleteExistingData && !SliderActions.ACTION_UPDATE.equals(action)) { + FileSystem dfs = FileSystem.get(new URI(getFsDefaultName()), config); + + SliderFileSystem sliderFileSystem = new SliderFileSystem(dfs, config); + Path clusterDir = sliderFileSystem.buildClusterDirPath(clustername); + LOG.info("deleting instance data at {}", clusterDir); + //this is a safety check to stop us doing something stupid like deleting / + assertTrue(clusterDir.toString().contains("/.slider/")); + rigorousDelete(sliderFileSystem, clusterDir, 60000); + } + + + List argsList = new ArrayList<>(); + argsList.addAll(Arrays.asList( + action, clustername, + Arguments.ARG_MANAGER, getRMAddr(), + Arguments.ARG_FILESYSTEM, getFsDefaultName(), + Arguments.ARG_DEBUG)); + + argsList.addAll(getExtraCLIArgs()); + + if (extraArgs != null) { + argsList.addAll(extraArgs); + } + ServiceLauncher launcher = launchClientAgainstMiniMR( + //config includes RM binding info + new YarnConfiguration(config), + //varargs list of command line params + argsList + ); + assertEquals(0, launcher.getServiceExitCode()); + SliderClient client = launcher.getService(); + if (blockUntilRunning) { + client.monitorAppToRunning(new Duration(CLUSTER_GO_LIVE_TIME)); + } + return launcher; + } + + /** * Delete with some pauses and backoff; designed to handle slow delete * operation in windows. */ @@ -652,28 +709,6 @@ public String getApplicationHome() { return getTestConfiguration().getTrimmed(getApplicationHomeKey()); } - public List getImageCommands() { - if (switchToImageDeploy) { - // its an image that had better be defined - assertNotNull(getArchivePath()); - if (!imageIsRemote) { - // its not remote, so assert it exists - File f = new File(getArchivePath()); - assertTrue(f.exists()); - return Arrays.asList(Arguments.ARG_IMAGE, f.toURI().toString()); - } else { - assertNotNull(remoteImageURI); - - // if it is remote, then its whatever the archivePath property refers to - return Arrays.asList(Arguments.ARG_IMAGE, remoteImageURI.toString()); - } - } else { - assertNotNull(getApplicationHome()); - assertTrue(new File(getApplicationHome()).exists()); - return Arrays.asList(Arguments.ARG_APP_HOME, getApplicationHome()); - } - } - /** * Get the resource configuration dir in the source tree. * diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/YarnZKMiniClusterTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/YarnZKMiniClusterTestBase.java index 322b346..3fe2942 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/YarnZKMiniClusterTestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/YarnZKMiniClusterTestBase.java @@ -109,9 +109,7 @@ protected String createMiniCluster(String name, int numLogDirs, boolean startZK, boolean startHDFS) throws IOException { - if (SliderUtils.isUnset(name)) { - name = methodName.getMethodName(); - } + name = buildClustername(name); createMicroZKCluster("-" + name, conf); conf.setBoolean(RegistryConstants.KEY_REGISTRY_ENABLED, true); conf.set(RegistryConstants.KEY_REGISTRY_ZK_QUORUM, getZKBinding()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app-override-resolved.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app-override-resolved.json deleted file mode 100644 index e2a21ea..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app-override-resolved.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "name": "app-1", - "lifetime": "3600", - "configuration": { - "properties": { - "g1": "a", - "g2": "b" - } - }, - "resource": { - "cpus": 1, - "memory": "512" - }, - "number_of_containers": 2, - "components": [ - { - "name": "simple", - "configuration": { - "properties": { - "g1": "a", - "g2": "b" - } - } - }, - { - "name": "master", - "configuration": { - "properties": { - "g1": "overridden", - "g2": "b" - } - } - }, - { - "name": "worker", - "resource": { - "cpus": 1, - "memory": "1024" - }, - "configuration": { - "properties": { - "g1": "overridden-by-worker", - "g2": "b", - "timeout": "1000" - } - } - } - ] -} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app-resolved.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app-resolved.json deleted file mode 100644 index cd1ab6f..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app-resolved.json +++ /dev/null @@ -1,81 +0,0 @@ -{ - "name": "zk-app-1", - "lifetime": "3600", - "configuration": { - "properties": { - "internal.chaos.monkey.interval.seconds": "60", - "zookeeper.port": "2181", - "zookeeper.path": "/yarnapps_small_cluster", - "zookeeper.hosts": "zoo1,zoo2,zoo3", - "env.MALLOC_ARENA_MAX": "4", - "site.hbase.master.startup.retainassign": "true", - "site.fs.defaultFS": "hdfs://cluster:8020", - "site.fs.default.name": "hdfs://cluster:8020", - "site.hbase.master.info.port": "0", - "site.hbase.regionserver.info.port": "0" - } - }, - "resource": { - "cpus": 1, - "memory": "512" - }, - "number_of_containers": 2, - "components": [ - { - "name": "simple", - "number_of_containers": 2, - "configuration": { - "properties": { - "g1": "a", - "g2": "b" - } - } - }, - { - "name": "master", - "number_of_containers": 1, - "resource": { - "cpus": 1, - "memory": "512" - }, - "configuration": { - "properties": { - "zookeeper.port": "2181", - "zookeeper.path": "/yarnapps_small_cluster", - "zookeeper.hosts": "zoo1,zoo2,zoo3", - "env.MALLOC_ARENA_MAX": "4", - "site.hbase.master.startup.retainassign": "true", - "site.fs.defaultFS": "hdfs://cluster:8020", - "site.fs.default.name": "hdfs://cluster:8020", - "site.hbase.master.info.port": "0", - "site.hbase.regionserver.info.port": "0", - "jvm.heapsize": "512M" - } - } - }, - { - "name": "worker", - "number_of_containers": 5, - "resource": { - "cpus": 1, - "memory": "1024" - }, - "configuration": { - "properties": { - "g1": "overridden-by-worker", - "g2": "b", - "zookeeper.port": "2181", - "zookeeper.path": "/yarnapps_small_cluster", - "zookeeper.hosts": "zoo1,zoo2,zoo3", - "env.MALLOC_ARENA_MAX": "4", - "site.hbase.master.startup.retainassign": "true", - "site.fs.defaultFS": "hdfs://cluster:8020", - "site.fs.default.name": "hdfs://cluster:8020", - "site.hbase.master.info.port": "0", - "site.hbase.regionserver.info.port": "0", - "jvm.heapsize": "512M" - } - } - } - ] -} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/default.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/default.json new file mode 100644 index 0000000..c4123cb --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/default.json @@ -0,0 +1,16 @@ +{ + "name": "default-app-1", + "lifetime": "3600", + "components" : + [ + { + "name": "SLEEP", + "number_of_containers": 1, + "launch_command": "sleep 3600", + "resource": { + "cpus": 2, + "memory": "256" + } + } + ] +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/external1.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/external1.json new file mode 100644 index 0000000..aeeb6a8 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/external1.json @@ -0,0 +1,21 @@ +{ + "name": "external-1", + "lifetime": "3600", + "components": [ + { + "name": "ext", + "artifact": { + "type": "APPLICATION", + "id": "app-1" + } + }, + { + "name": "other", + "number_of_containers": 2, + "resource": { + "cpus": 1, + "memory": "512" + } + } + ] +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/external2.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/external2.json new file mode 100644 index 0000000..5ab26cc --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/external2.json @@ -0,0 +1,21 @@ +{ + "name": "external-2", + "lifetime": "3600", + "components": [ + { + "name": "ext", + "artifact": { + "type": "APPLICATION", + "id": "external-1" + } + }, + { + "name": "another", + "number_of_containers": 1, + "resource": { + "cpus": 1, + "memory": "512" + } + } + ] +} \ No newline at end of file