diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/ApplicationApi.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/ApplicationApi.java index 0fb6402..0f4bdae 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/ApplicationApi.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/ApplicationApi.java @@ -30,8 +30,6 @@ Response getApplications(String state); - Response getApplication(String appName); - Response deleteApplication(String appName); Response updateApplication(String appName, Application updateAppData); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java index 5a4726e..48429d0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java @@ -17,103 +17,60 @@ package org.apache.hadoop.yarn.services.api.impl; -import static org.apache.hadoop.yarn.services.utils.RestApiConstants.*; -import static org.apache.hadoop.yarn.services.utils.RestApiErrorMessages.*; - -import java.io.FileNotFoundException; -import java.io.IOException; -import java.lang.reflect.UndeclaredThrowableException; -import java.security.PrivilegedExceptionAction; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; -import java.util.regex.Pattern; - -import javax.ws.rs.Consumes; -import javax.ws.rs.DELETE; -import javax.ws.rs.GET; -import javax.ws.rs.POST; -import javax.ws.rs.PUT; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.Produces; -import javax.ws.rs.QueryParam; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; -import javax.ws.rs.core.Response.Status; - -import org.apache.commons.lang.SerializationUtils; +import com.google.common.annotations.VisibleForTesting; +import com.google.inject.Singleton; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.services.api.ApplicationApi; import org.apache.slider.api.resource.Application; import org.apache.slider.api.resource.ApplicationState; import org.apache.slider.api.resource.ApplicationStatus; import org.apache.slider.api.resource.Artifact; import org.apache.slider.api.resource.Component; -import org.apache.slider.api.resource.ConfigFile; import org.apache.slider.api.resource.Configuration; -import org.apache.slider.api.resource.Container; -import org.apache.slider.api.resource.ContainerState; import org.apache.slider.api.resource.Resource; -import org.apache.slider.api.OptionKeys; -import org.apache.slider.api.ResourceKeys; -import org.apache.slider.api.StateValues; import org.apache.slider.client.SliderClient; -import org.apache.slider.common.SliderExitCodes; -import org.apache.slider.common.SliderKeys; -import org.apache.slider.common.params.ActionCreateArgs; -import org.apache.slider.common.params.ActionFlexArgs; import org.apache.slider.common.params.ActionFreezeArgs; -import org.apache.slider.common.params.ActionListArgs; -import org.apache.slider.common.params.ActionRegistryArgs; -import org.apache.slider.common.params.ActionThawArgs; -import org.apache.slider.common.params.ActionUpdateArgs; -import org.apache.slider.common.params.ComponentArgsDelegate; import org.apache.slider.common.tools.SliderUtils; import org.apache.slider.common.tools.SliderVersionInfo; import org.apache.slider.core.buildutils.BuildHelper; -import org.apache.slider.core.exceptions.BadClusterStateException; -import org.apache.slider.core.exceptions.NotFoundException; import org.apache.slider.core.exceptions.SliderException; -import org.apache.slider.core.exceptions.UnknownApplicationInstanceException; -import org.apache.slider.core.registry.docstore.ConfigFormat; -import org.apache.slider.providers.docker.DockerKeys; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.annotations.VisibleForTesting; -import com.google.gson.JsonElement; -import com.google.gson.JsonNull; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; -import com.google.inject.Singleton; +import javax.ws.rs.Consumes; +import javax.ws.rs.DELETE; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.PUT; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.Response.Status; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import static org.apache.hadoop.yarn.services.utils.RestApiConstants.*; +import static org.apache.hadoop.yarn.services.utils.RestApiErrorMessages.*; @Singleton @Path(APPLICATIONS_API_RESOURCE_PATH) @Consumes({ MediaType.APPLICATION_JSON }) @Produces({ MediaType.APPLICATION_JSON }) -public class ApplicationApiService implements ApplicationApi { - private static final Logger logger = LoggerFactory - .getLogger(ApplicationApiService.class); +public class ApplicationApiService { + private static final Logger logger = LoggerFactory.getLogger(ApplicationApiService.class); private static org.apache.hadoop.conf.Configuration SLIDER_CONFIG; private static UserGroupInformation SLIDER_USER; private static SliderClient SLIDER_CLIENT; private static Response SLIDER_VERSION; - private static final JsonParser JSON_PARSER = new JsonParser(); - private static final JsonObject EMPTY_JSON_OBJECT = new JsonObject(); - private static final ActionListArgs ACTION_LIST_ARGS = new ActionListArgs(); private static final ActionFreezeArgs ACTION_FREEZE_ARGS = new ActionFreezeArgs(); static { @@ -131,8 +88,7 @@ protected static void init() { @GET @Path("/versions/slider-version") @Consumes({ MediaType.APPLICATION_JSON }) - @Produces({ MediaType.APPLICATION_JSON }) - public Response getSliderVersion() { + @Produces({ MediaType.APPLICATION_JSON }) public Response getSliderVersion() { logger.info("GET: getSliderVersion"); return SLIDER_VERSION; } @@ -148,59 +104,41 @@ private static Response initSliderVersion() { + "\", \"hadoop_version\": \"" + hadoopVersion + "\"}").build(); } - @POST - @Consumes({ MediaType.APPLICATION_JSON }) + @POST @Consumes({ MediaType.APPLICATION_JSON }) @Produces({ MediaType.APPLICATION_JSON }) public Response createApplication(Application application) { - logger.info("POST: createApplication for app = {}", application); + logger.info("POST: createApplication = {}", application); ApplicationStatus applicationStatus = new ApplicationStatus(); - Map compNameArtifactIdMap = new HashMap<>(); // post payload validation try { - validateApplicationPostPayload(application, compNameArtifactIdMap); + validateApplicationPostPayload(application); } catch (IllegalArgumentException e) { applicationStatus.setDiagnostics(e.getMessage()); return Response.status(Status.BAD_REQUEST).entity(applicationStatus) .build(); } - String applicationId = null; + ApplicationId applicationId = null; try { - applicationId = createSliderApp(application, compNameArtifactIdMap); + applicationId = SLIDER_CLIENT.actionCreate(application); + logger.info("Successfully created application " + application.getName() + + " applicationId = " + applicationId); applicationStatus.setState(ApplicationState.ACCEPTED); - } catch (SliderException se) { - logger.error("Create application failed", se); - if (se.getExitCode() == SliderExitCodes.EXIT_APPLICATION_IN_USE) { - applicationStatus.setDiagnostics(ERROR_APPLICATION_IN_USE); - return Response.status(Status.BAD_REQUEST).entity(applicationStatus) - .build(); - } else if (se.getExitCode() == SliderExitCodes.EXIT_INSTANCE_EXISTS) { - applicationStatus.setDiagnostics(ERROR_APPLICATION_INSTANCE_EXISTS); - return Response.status(Status.BAD_REQUEST).entity(applicationStatus) - .build(); - } else { - applicationStatus.setDiagnostics(se.getMessage()); - } + applicationStatus.setUri( + CONTEXT_ROOT + APPLICATIONS_API_RESOURCE_PATH + "/" + application + .getName()); + return Response.status(Status.CREATED).entity(applicationStatus).build(); } catch (Exception e) { - logger.error("Create application failed", e); - applicationStatus.setDiagnostics(e.getMessage()); - } - - if (StringUtils.isNotEmpty(applicationId)) { - applicationStatus.setUri(CONTEXT_ROOT + APPLICATIONS_API_RESOURCE_PATH - + "/" + application.getName()); - // 202 = ACCEPTED - return Response.status(HTTP_STATUS_CODE_ACCEPTED) - .entity(applicationStatus).build(); - } else { + String message = "Failed to create application " + application.getName(); + logger.error(message, e); + applicationStatus.setDiagnostics(message + ": " + e.getMessage()); return Response.status(Status.INTERNAL_SERVER_ERROR) .entity(applicationStatus).build(); } } @VisibleForTesting - protected void validateApplicationPostPayload(Application application, - Map compNameArtifactIdMap) { + protected void validateApplicationPostPayload(Application application) { if (StringUtils.isEmpty(application.getName())) { throw new IllegalArgumentException(ERROR_APPLICATION_NAME_INVALID); } @@ -224,13 +162,10 @@ protected void validateApplicationPostPayload(Application application, if (application.getConfiguration() == null) { application.setConfiguration(new Configuration()); } - addPropertyToConfiguration(application.getConfiguration(), - SliderKeys.COMPONENT_TYPE_KEY, - SliderKeys.COMPONENT_TYPE_EXTERNAL_APP); } // resource - validateApplicationResource(application.getResource(), null, application - .getArtifact().getType()); + validateApplicationResource(application.getResource(), null, + application.getArtifact().getType()); // container size if (application.getNumberOfContainers() == null) { @@ -238,7 +173,7 @@ protected void validateApplicationPostPayload(Application application, } // Since it is a simple app with no components, create a default component - application.setComponents(getDefaultComponentAsList(application)); + application.getComponents().add(createDefaultComponent(application)); } else { // If the application has components, then run checks for each component. // Let global values take effect if component level values are not @@ -253,12 +188,12 @@ protected void validateApplicationPostPayload(Application application, } // If still null raise validation exception if (comp.getArtifact() == null) { - throw new IllegalArgumentException(String.format( - ERROR_ARTIFACT_FOR_COMP_INVALID, comp.getName())); + throw new IllegalArgumentException( + String.format(ERROR_ARTIFACT_FOR_COMP_INVALID, comp.getName())); } if (StringUtils.isEmpty(comp.getArtifact().getId())) { - throw new IllegalArgumentException(String.format( - ERROR_ARTIFACT_ID_FOR_COMP_INVALID, comp.getName())); + throw new IllegalArgumentException(String + .format(ERROR_ARTIFACT_ID_FOR_COMP_INVALID, comp.getName())); } // If artifact is of type APPLICATION, add a slider specific property @@ -266,10 +201,6 @@ protected void validateApplicationPostPayload(Application application, if (comp.getConfiguration() == null) { comp.setConfiguration(new Configuration()); } - addPropertyToConfiguration(comp.getConfiguration(), - SliderKeys.COMPONENT_TYPE_KEY, - SliderKeys.COMPONENT_TYPE_EXTERNAL_APP); - compNameArtifactIdMap.put(comp.getName(), comp.getArtifact().getId()); comp.setName(comp.getArtifact().getId()); } @@ -277,16 +208,16 @@ protected void validateApplicationPostPayload(Application application, if (comp.getResource() == null) { comp.setResource(globalResource); } - validateApplicationResource(comp.getResource(), comp, comp - .getArtifact().getType()); + validateApplicationResource(comp.getResource(), comp, + comp.getArtifact().getType()); // container count if (comp.getNumberOfContainers() == null) { comp.setNumberOfContainers(globalNumberOfContainers); } if (comp.getNumberOfContainers() == null) { - throw new IllegalArgumentException(String.format( - ERROR_CONTAINERS_COUNT_FOR_COMP_INVALID, comp.getName())); + throw new IllegalArgumentException(String + .format(ERROR_CONTAINERS_COUNT_FOR_COMP_INVALID, comp.getName())); } } } @@ -304,17 +235,16 @@ private void validateApplicationResource(Resource resource, Component comp, return; } if (resource == null) { - throw new IllegalArgumentException(comp == null ? ERROR_RESOURCE_INVALID - : String.format(ERROR_RESOURCE_FOR_COMP_INVALID, comp.getName())); + throw new IllegalArgumentException(comp == null ? ERROR_RESOURCE_INVALID : + String.format(ERROR_RESOURCE_FOR_COMP_INVALID, comp.getName())); } // One and only one of profile OR cpus & memory can be specified. Specifying // both raises validation error. - if (StringUtils.isNotEmpty(resource.getProfile()) - && (resource.getCpus() != null - || StringUtils.isNotEmpty(resource.getMemory()))) { + if (StringUtils.isNotEmpty(resource.getProfile()) && (resource.getCpus() != null + || StringUtils.isNotEmpty(resource.getMemory()))) { throw new IllegalArgumentException( - comp == null ? ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_NOT_SUPPORTED - : String.format( + comp == null ? ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_NOT_SUPPORTED : + String.format( ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_FOR_COMP_NOT_SUPPORTED, comp.getName())); } @@ -329,345 +259,21 @@ private void validateApplicationResource(Resource resource, Component comp, Integer cpus = resource.getCpus(); if (StringUtils.isEmpty(memory)) { throw new IllegalArgumentException( - comp == null ? ERROR_RESOURCE_MEMORY_INVALID : String.format( - ERROR_RESOURCE_MEMORY_FOR_COMP_INVALID, comp.getName())); + comp == null ? ERROR_RESOURCE_MEMORY_INVALID : String + .format(ERROR_RESOURCE_MEMORY_FOR_COMP_INVALID, comp.getName())); } if (cpus == null) { throw new IllegalArgumentException( - comp == null ? ERROR_RESOURCE_CPUS_INVALID : String.format( - ERROR_RESOURCE_CPUS_FOR_COMP_INVALID, comp.getName())); + comp == null ? ERROR_RESOURCE_CPUS_INVALID : String + .format(ERROR_RESOURCE_CPUS_FOR_COMP_INVALID, comp.getName())); } if (cpus <= 0) { throw new IllegalArgumentException( - comp == null ? ERROR_RESOURCE_CPUS_INVALID_RANGE : String.format( - ERROR_RESOURCE_CPUS_FOR_COMP_INVALID_RANGE, comp.getName())); + comp == null ? ERROR_RESOURCE_CPUS_INVALID_RANGE : String + .format(ERROR_RESOURCE_CPUS_FOR_COMP_INVALID_RANGE, comp.getName())); } } - private String createSliderApp(Application application, - Map compNameArtifactIdMap) throws IOException, - YarnException, InterruptedException { - final String appName = application.getName(); - final String queueName = application.getQueue(); - - final ActionCreateArgs createArgs = new ActionCreateArgs(); - addAppConfOptions(createArgs, application, compNameArtifactIdMap); - addResourceOptions(createArgs, application); - - createArgs.provider = DockerKeys.PROVIDER_DOCKER; - - if (queueName != null && queueName.trim().length() > 0) { - createArgs.queue = queueName.trim(); - } - createArgs.lifetime = application.getLifetime(); - return invokeSliderClientRunnable(new SliderClientContextRunnable() { - @Override - public String run(SliderClient sliderClient) throws YarnException, - IOException, InterruptedException { - sliderClient.actionCreate(appName, createArgs); - ApplicationId applicationId = sliderClient.applicationId; - if (applicationId != null) { - return applicationId.toString(); - // return getApplicationIdString(applicationId); - } - return null; - } - }); - } - - private void addAppConfOptions(ActionCreateArgs createArgs, - Application application, Map compNameArtifactIdMap) throws IOException { - List appCompOptionTriples = createArgs.optionsDelegate.compOptTriples; // TODO: optionTuples instead of compOptTriples - logger.info("Initial appCompOptionTriples = {}", - Arrays.toString(appCompOptionTriples.toArray())); - List appOptions = createArgs.optionsDelegate.optionTuples; - logger.info("Initial appOptions = {}", - Arrays.toString(appOptions.toArray())); - // TODO: Set Slider-AM memory and vcores here - // appCompOptionTriples.addAll(Arrays.asList(SLIDER_APPMASTER_COMPONENT_NAME, - // "", "")); - - // Global configuration - for override purpose - // TODO: add it to yaml - Configuration globalConfig = null; - // Configuration globalConfig = (Configuration) SerializationUtils - // .clone(application.getConfiguration()); - - // TODO: Add the below into globalConfig - // if (application.getConfigurations() != null) { - // for (Entry entry : application.getConfigurations() - // .entrySet()) { - // globalConf.addProperty(entry.getKey(), entry.getValue()); - // } - // } - - Set uniqueGlobalPropertyCache = new HashSet<>(); - if (application.getConfiguration() != null) { - if (application.getConfiguration().getProperties() != null) { - for (Map.Entry propEntry : application - .getConfiguration().getProperties().entrySet()) { - addOptionsIfNotPresent(appOptions, uniqueGlobalPropertyCache, - propEntry.getKey(), propEntry.getValue()); - } - } - List configFiles = application.getConfiguration().getFiles(); - if (configFiles != null && !configFiles.isEmpty()) { - addOptionsIfNotPresent(appOptions, uniqueGlobalPropertyCache, - SliderKeys.AM_CONFIG_GENERATION, "true"); - for (ConfigFile configFile : configFiles) { - addOptionsIfNotPresent(appOptions, uniqueGlobalPropertyCache, - OptionKeys.CONF_FILE_PREFIX + configFile.getSrcFile() + - OptionKeys.NAME_SUFFIX, configFile.getDestFile()); - addOptionsIfNotPresent(appOptions, uniqueGlobalPropertyCache, - OptionKeys.CONF_FILE_PREFIX + configFile.getSrcFile() + - OptionKeys.TYPE_SUFFIX, configFile.getType().toString()); - } - } - } - if (application.getComponents() != null) { - - Map appQuicklinks = application.getQuicklinks(); - if (appQuicklinks != null) { - for (Map.Entry quicklink : appQuicklinks.entrySet()) { - addOptionsIfNotPresent(appOptions, uniqueGlobalPropertyCache, - OptionKeys.EXPORT_PREFIX + quicklink.getKey(), - quicklink.getValue()); - } - } - - Map placeholders = new HashMap<>(); - placeholders.put(PLACEHOLDER_APP_NAME, application.getName()); - for (Component comp : application.getComponents()) { - placeholders.put(PLACEHOLDER_APP_COMPONENT_NAME, comp.getName()); - if (comp.getArtifact().getType() == Artifact.TypeEnum.DOCKER) { - appCompOptionTriples.addAll(Arrays.asList(comp.getName(), - DockerKeys.DOCKER_IMAGE, comp.getArtifact().getId() == null ? - application.getArtifact().getId() : comp.getArtifact().getId())); - appCompOptionTriples.addAll(Arrays.asList(comp.getName(), - DockerKeys.DOCKER_START_COMMAND, comp.getLaunchCommand() == null ? - replacePlaceholders(application.getLaunchCommand(), placeholders) - : replacePlaceholders(comp.getLaunchCommand(), placeholders))); - appCompOptionTriples.addAll(Arrays.asList(comp.getName(), - DockerKeys.DOCKER_NETWORK, DockerKeys.DEFAULT_DOCKER_NETWORK)); - if (comp.getRunPrivilegedContainer() != null) { - appCompOptionTriples.addAll(Arrays.asList(comp.getName(), - DockerKeys.DOCKER_USE_PRIVILEGED, - comp.getRunPrivilegedContainer().toString())); - } - } - - if (comp.getConfiguration() != null) { - List configFiles = comp.getConfiguration().getFiles(); - if (configFiles != null && !configFiles.isEmpty()) { - appCompOptionTriples.addAll(Arrays.asList(comp.getName(), - SliderKeys.AM_CONFIG_GENERATION, "true")); - for (ConfigFile configFile : configFiles) { - appCompOptionTriples.addAll(Arrays.asList(comp.getName(), - OptionKeys.CONF_FILE_PREFIX + configFile.getSrcFile() + - OptionKeys.NAME_SUFFIX, configFile.getDestFile())); - appCompOptionTriples.addAll(Arrays.asList(comp.getName(), - OptionKeys.CONF_FILE_PREFIX + configFile.getSrcFile() + - OptionKeys.TYPE_SUFFIX, configFile.getType().toString())); - } - } - } - - if (Boolean.TRUE.equals(comp.getUniqueComponentSupport())) { - for (int i = 1; i <= comp.getNumberOfContainers(); i++) { - placeholders.put(PLACEHOLDER_COMPONENT_ID, Integer.toString(i)); - appCompOptionTriples.addAll(createAppConfigComponent( - comp.getName() + i, comp, comp.getName() + i, globalConfig, - placeholders, compNameArtifactIdMap)); - } - } else { - appCompOptionTriples.addAll(createAppConfigComponent(comp.getName(), - comp, comp.getName(), globalConfig, null, compNameArtifactIdMap)); - } - } - } - - logger.info("Updated appCompOptionTriples = {}", - Arrays.toString(appCompOptionTriples.toArray())); - logger.info("Updated appOptions = {}", - Arrays.toString(appOptions.toArray())); - } - - private void addOptionsIfNotPresent(List options, - Set uniqueGlobalPropertyCache, String key, String value) { - if (uniqueGlobalPropertyCache == null) { - options.addAll(Arrays.asList(key, value)); - } else if (!uniqueGlobalPropertyCache.contains(key)) { - options.addAll(Arrays.asList(key, value)); - uniqueGlobalPropertyCache.add(key); - } - } - - private void addPropertyToConfiguration(Configuration conf, String key, - String value) { - if (conf == null) { - return; - } - if (conf.getProperties() == null) { - conf.setProperties(new HashMap()); - } - conf.getProperties().put(key, value); - } - - private List createAppConfigComponent(String compName, - Component component, String configPrefix, Configuration globalConf, - Map placeholders, - Map compNameArtifactIdMap) { - List appConfOptTriples = new ArrayList<>(); - - if (component.getConfiguration() != null - && component.getConfiguration().getProperties() != null) { - for (Map.Entry propEntry : component.getConfiguration() - .getProperties().entrySet()) { - appConfOptTriples.addAll(Arrays.asList(compName, propEntry.getKey(), - replacePlaceholders(propEntry.getValue(), placeholders))); - } - } - - // If artifact is of type APPLICATION, then in the POST JSON there will - // be no component definition for that artifact. Hence it's corresponding id - // field is added. Every external APPLICATION has a unique id field. - List convertedDeps = new ArrayList<>(); - for (String dep : component.getDependencies()) { - if (compNameArtifactIdMap.containsKey(dep)) { - convertedDeps.add(compNameArtifactIdMap.get(dep)); - } else { - convertedDeps.add(dep); - } - } - // If the DNS dependency property is set to true for a component, it means - // that it is ensured that DNS entry has been added for all the containers - // of this component, before moving on to the next component in the DAG. - if (hasPropertyWithValue(component, PROPERTY_DNS_DEPENDENCY, "true")) { - if (component.getArtifact().getType() == Artifact.TypeEnum.APPLICATION) { - convertedDeps.add(component.getArtifact().getId()); - } else { - convertedDeps.add(compName); - } - } - if (convertedDeps.size() > 0) { - appConfOptTriples.addAll(Arrays.asList(compName, "requires", - StringUtils.join(convertedDeps, ","))); - } - return appConfOptTriples; - } - - private String replacePlaceholders(String value, - Map placeholders) { - if (StringUtils.isEmpty(value) || placeholders == null) { - return value; - } - for (Map.Entry placeholder : placeholders.entrySet()) { - value = value.replaceAll(Pattern.quote(placeholder.getKey()), - placeholder.getValue()); - } - return value; - } - - private List createAppConfigGlobal(Component component, - Configuration globalConf, Set uniqueGlobalPropertyCache) { - List appOptions = new ArrayList<>(); - if (component.getConfiguration() != null - && component.getConfiguration().getProperties() != null) { - for (Map.Entry propEntry : component.getConfiguration() - .getProperties().entrySet()) { - addOptionsIfNotPresent(appOptions, uniqueGlobalPropertyCache, - propEntry.getKey(), propEntry.getValue()); - } - } - return appOptions; - } - - private void addResourceOptions(ActionCreateArgs createArgs, - Application application) throws IOException { - List resCompOptionTriples = createArgs.optionsDelegate.resCompOptTriples; - logger.info("Initial resCompOptTriples = {}", - Arrays.toString(resCompOptionTriples.toArray())); - // TODO: Add any Slider AM resource specific props here like jvm.heapsize - // resCompOptionTriples.addAll(Arrays.asList(SLIDER_APPMASTER_COMPONENT_NAME, - // "", "")); - - // Global resource - for override purpose - Resource globalResource = (Resource) SerializationUtils.clone(application - .getResource()); - // Priority seeded with 1, expecting every new component will increase it by - // 1 making it ready for the next component to use. - if (application.getComponents() != null) { - int priority = 1; - for (Component comp : application.getComponents()) { - if (hasPropertyWithValue(comp, SliderKeys.COMPONENT_TYPE_KEY, - SliderKeys.COMPONENT_TYPE_EXTERNAL_APP)) { - continue; - } - if (Boolean.TRUE.equals(comp.getUniqueComponentSupport())) { - for (int i = 1; i <= comp.getNumberOfContainers(); i++) { - resCompOptionTriples.addAll(createResourcesComponent(comp.getName() - + i, comp, priority, 1, globalResource)); - priority++; - } - } else { - resCompOptionTriples.addAll(createResourcesComponent(comp.getName(), - comp, priority, comp.getNumberOfContainers(), globalResource)); - priority++; - } - } - } - - logger.info("Updated resCompOptTriples = {}", - Arrays.toString(resCompOptionTriples.toArray())); - } - - private boolean hasPropertyWithValue(Component comp, String key, String value) { - if (comp == null || key == null) { - return false; - } - if (comp.getConfiguration() == null - || comp.getConfiguration().getProperties() == null) { - return false; - } - Map props = comp.getConfiguration().getProperties(); - if (props.containsKey(key)) { - if (value == null) { - return props.get(key) == null; - } else { - if (value.equals(props.get(key))) { - return true; - } - } - } - return false; - } - - private List createResourcesComponent(String compName, - Component component, int priority, long numInstances, - Resource globalResource) { - String memory = component.getResource() == null ? globalResource - .getMemory() : component.getResource().getMemory(); - Integer cpus = component.getResource() == null ? globalResource.getCpus() - : component.getResource().getCpus(); - - List resCompOptTriples = new ArrayList(); - resCompOptTriples.addAll(Arrays.asList(compName, - ResourceKeys.COMPONENT_PRIORITY, Integer.toString(priority))); - resCompOptTriples.addAll(Arrays.asList(compName, - ResourceKeys.COMPONENT_INSTANCES, Long.toString(numInstances))); - resCompOptTriples.addAll(Arrays.asList(compName, ResourceKeys.YARN_MEMORY, - memory)); - resCompOptTriples.addAll(Arrays.asList(compName, ResourceKeys.YARN_CORES, - cpus.toString())); - if (component.getPlacementPolicy() != null) { - resCompOptTriples.addAll(Arrays.asList(compName, - ResourceKeys.COMPONENT_PLACEMENT_POLICY, - component.getPlacementPolicy().getLabel())); - } - - return resCompOptTriples; - } - private static UserGroupInformation getSliderUser() { if (SLIDER_USER != null) { return SLIDER_USER; @@ -686,42 +292,19 @@ private static UserGroupInformation getSliderUser() { return sliderUser; } - private T invokeSliderClientRunnable( - final SliderClientContextRunnable runnable) - throws IOException, InterruptedException, YarnException { - try { - T value = SLIDER_USER.doAs(new PrivilegedExceptionAction() { - @Override - public T run() throws Exception { - return runnable.run(SLIDER_CLIENT); - } - }); - return value; - } catch (UndeclaredThrowableException e) { - Throwable cause = e.getCause(); - if (cause instanceof YarnException) { - YarnException ye = (YarnException) cause; - throw ye; - } - throw e; - } - } protected static SliderClient createSliderClient() { if (SLIDER_CLIENT != null) { return SLIDER_CLIENT; } - org.apache.hadoop.conf.Configuration sliderClientConfiguration = SLIDER_CONFIG; + org.apache.hadoop.conf.Configuration sliderClientConfiguration = + SLIDER_CONFIG; SliderClient client = new SliderClient() { - @Override - public void init(org.apache.hadoop.conf.Configuration conf) { + @Override public void init(org.apache.hadoop.conf.Configuration conf) { super.init(conf); try { initHadoopBinding(); - } catch (SliderException e) { - throw new RuntimeException( - "Unable to automatically init Hadoop binding", e); - } catch (IOException e) { + } catch (SliderException | IOException e) { throw new RuntimeException( "Unable to automatically init Hadoop binding", e); } @@ -730,8 +313,7 @@ public void init(org.apache.hadoop.conf.Configuration conf) { try { logger .debug("Slider Client configuration: {}", sliderClientConfiguration); - sliderClientConfiguration = client.bindArgs(sliderClientConfiguration, - new String[] { "help" }); + sliderClientConfiguration = client.bindArgs(sliderClientConfiguration, new String[] { "help" }); client.init(sliderClientConfiguration); client.start(); } catch (Exception e) { @@ -743,9 +325,9 @@ public void init(org.apache.hadoop.conf.Configuration conf) { private static String getUserToRunAs() { String user = System.getenv(PROPERTY_APP_RUNAS_USER); - if (StringUtils.isEmpty(user)) { - user = "root"; - } + // if (StringUtils.isEmpty(user)) { + // user = "root"; + // } return user; } @@ -760,589 +342,116 @@ private static String getUserToRunAs() { return yarnConfig; } - private interface SliderClientContextRunnable { - T run(SliderClient sliderClient) - throws YarnException, IOException, InterruptedException; - } - - @GET - @Consumes({ MediaType.APPLICATION_JSON }) - @Produces({ MediaType.APPLICATION_JSON }) - public Response getApplications(@QueryParam("state") String state) { - logger.info("GET: getApplications with param state = {}", state); - - // Get all applications in a specific state - lighter projection. For full - // detail, call getApplication on a specific app. - Set applications; - try { - if (StringUtils.isNotEmpty(state)) { - ApplicationStatus appStatus = new ApplicationStatus(); - try { - ApplicationState.valueOf(state); - } catch (IllegalArgumentException e) { - appStatus.setDiagnostics("Invalid value for param state - " + state); - return Response.status(Status.BAD_REQUEST).entity(appStatus).build(); - } - applications = getSliderApplications(state); - } else { - applications = getSliderApplications(true); - } - } catch (Exception e) { - logger.error("Get applications failed", e); - return Response.status(Status.INTERNAL_SERVER_ERROR).build(); - } - - Set apps = new HashSet(); - if (applications.size() > 0) { - try { - for (ApplicationReport app : applications) { - Application application = new Application(); - application.setLifetime(app.getApplicationTimeouts().get( - ApplicationTimeoutType.LIFETIME).getRemainingTime()); - application.setLaunchTime(new Date(app.getStartTime())); - application.setName(app.getName()); - // Containers not required, setting to null to avoid empty list - application.setContainers(null); - apps.add(application); - } - } catch (Exception e) { - logger.error("Get applications failed", e); - return Response.status(Status.INTERNAL_SERVER_ERROR).build(); - } - } - - return Response.ok().entity(apps).build(); - } + // The information this REST endpoint currently returned can be retrieved from + // RM web services + // Probably the data from AM is more important. Do that later. +// @GET @Consumes({ MediaType.APPLICATION_JSON }) +// @Produces({ MediaType.APPLICATION_JSON }) +// public Response getApplications(@QueryParam("state") String state) { +// logger.info("GET: getApplications with param state = {}", state); +// return null; +// } - @GET - @Path("/{app_name}") + @GET @Path("/{app_name}") @Consumes({ MediaType.APPLICATION_JSON }) @Produces({ MediaType.APPLICATION_JSON }) public Response getApplication(@PathParam("app_name") String appName) { logger.info("GET: getApplication for appName = {}", appName); + ApplicationStatus applicationStatus = new ApplicationStatus(); // app name validation if (!SliderUtils.isClusternameValid(appName)) { - ApplicationStatus applicationStatus = new ApplicationStatus(); - applicationStatus.setDiagnostics("Invalid application name"); + applicationStatus.setDiagnostics("Invalid application name: " + appName); applicationStatus.setCode(ERROR_CODE_APP_NAME_INVALID); return Response.status(Status.NOT_FOUND).entity(applicationStatus) .build(); } - // Check if app exists try { - int livenessCheck = getSliderList(appName); - if (livenessCheck < 0) { - logger.info("Application not running"); - ApplicationStatus applicationStatus = new ApplicationStatus(); - applicationStatus.setDiagnostics(ERROR_APPLICATION_NOT_RUNNING); - applicationStatus.setCode(ERROR_CODE_APP_IS_NOT_RUNNING); + Application app = SLIDER_CLIENT.actionStatus(appName); + ApplicationReport report = SLIDER_CLIENT.findInstance(appName); + if (app != null && report != null) { + app.setLifetime( + report.getApplicationTimeouts().get(ApplicationTimeoutType.LIFETIME) + .getRemainingTime()); + logger.info("Application = {}", app); + return Response.ok(app).build(); + } else { + String message = "Application " + appName + " does not exist."; + logger.info(message); + applicationStatus.setCode(ERROR_CODE_APP_DOES_NOT_EXIST); + applicationStatus.setDiagnostics(message); return Response.status(Status.NOT_FOUND).entity(applicationStatus) .build(); } - } catch (UnknownApplicationInstanceException e) { - logger.error("Get application failed, application not found", e); - ApplicationStatus applicationStatus = new ApplicationStatus(); - applicationStatus.setDiagnostics(ERROR_APPLICATION_DOES_NOT_EXIST); - applicationStatus.setCode(ERROR_CODE_APP_DOES_NOT_EXIST); - return Response.status(Status.NOT_FOUND).entity(applicationStatus) - .build(); - } catch (Exception e) { - logger.error("Get application failed, application not running", e); - ApplicationStatus applicationStatus = new ApplicationStatus(); - applicationStatus.setDiagnostics(ERROR_APPLICATION_NOT_RUNNING); - applicationStatus.setCode(ERROR_CODE_APP_IS_NOT_RUNNING); - return Response.status(Status.NOT_FOUND).entity(applicationStatus) - .build(); - } - - Application app = new Application(); - app.setName(appName); - app.setUri(CONTEXT_ROOT + APPLICATIONS_API_RESOURCE_PATH + "/" - + appName); - // TODO: add status - app.setState(ApplicationState.ACCEPTED); - JsonObject appStatus = null; - JsonObject appRegistryQuicklinks = null; - try { - appStatus = getSliderApplicationStatus(appName); - appRegistryQuicklinks = getSliderApplicationRegistry(appName, - "quicklinks"); - return populateAppData(app, appStatus, appRegistryQuicklinks); - } catch (BadClusterStateException | NotFoundException e) { - logger.error( - "Get application failed, application not in running state yet", e); - ApplicationStatus applicationStatus = new ApplicationStatus(); - applicationStatus.setDiagnostics("Application not running yet"); - applicationStatus.setCode(ERROR_CODE_APP_SUBMITTED_BUT_NOT_RUNNING_YET); - return Response.status(Status.NOT_FOUND).entity(applicationStatus) - .build(); } catch (Exception e) { logger.error("Get application failed", e); - ApplicationStatus applicationStatus = new ApplicationStatus(); - applicationStatus.setDiagnostics("Failed to retrieve application: " - + e.getMessage()); + applicationStatus + .setDiagnostics("Failed to retrieve application: " + e.getMessage()); return Response.status(Status.INTERNAL_SERVER_ERROR) .entity(applicationStatus).build(); } } - private Response populateAppData(Application app, JsonObject appStatus, - JsonObject appRegistryQuicklinks) { - String appName = jsonGetAsString(appStatus, "name"); - Long totalNumberOfRunningContainers = 0L; - Long totalExpectedNumberOfRunningContainers = 0L; - Long totalNumberOfIpAssignedContainers = 0L; - - // info - JsonObject applicationInfo = jsonGetAsObject(appStatus, "info"); - if (applicationInfo != null) { - String applicationId = jsonGetAsString(applicationInfo, "info.am.app.id"); - if (applicationId != null) { - app.setId(applicationId); - } - } - - // state - String appState = jsonGetAsString(appStatus, "state"); - if (appState == null) { - // consider that app is still in ACCEPTED state - appState = String.valueOf(StateValues.STATE_INCOMPLETE); - } - switch (Integer.parseInt(appState)) { - case StateValues.STATE_LIVE: - app.setState(ApplicationState.STARTED); - break; - case StateValues.STATE_CREATED: - case StateValues.STATE_INCOMPLETE: - case StateValues.STATE_SUBMITTED: - app.setState(ApplicationState.ACCEPTED); - return Response.ok(app).build(); - case StateValues.STATE_DESTROYED: - case StateValues.STATE_STOPPED: - app.setState(ApplicationState.STOPPED); - return Response.ok(app).build(); - default: - break; - } - - // start time - app.setLaunchTime(appStatus.get("createTime") == null ? null - : new Date(appStatus.get("createTime").getAsLong())); - - app.setLifetime(queryLifetime(appName)); - - // Quicklinks - Map appQuicklinks = new HashMap<>(); - for (Map.Entry quicklink : appRegistryQuicklinks - .entrySet()) { - appQuicklinks.put(quicklink.getKey(), quicklink.getValue() == null ? null - : quicklink.getValue().getAsString()); - } - if (!appQuicklinks.isEmpty()) { - app.setQuicklinks(appQuicklinks); - } - - ArrayList componentNames = new ArrayList<>(); - - // status.live - JsonObject applicationStatus = jsonGetAsObject(appStatus, "status"); - // roles - JsonObject applicationRoles = jsonGetAsObject(appStatus, "roles"); - // statistics - JsonObject applicationStatistics = jsonGetAsObject(appStatus, "statistics"); - if (applicationRoles == null) { - // initialize to empty object to avoid too many null checks - applicationRoles = EMPTY_JSON_OBJECT; - } - if (applicationStatus != null) { - JsonObject applicationLive = jsonGetAsObject(applicationStatus, "live"); - if (applicationLive != null) { - for (Entry entry : applicationLive.entrySet()) { - if (entry.getKey().equals(SLIDER_APPMASTER_COMPONENT_NAME)) { - continue; - } - componentNames.add(entry.getKey()); - JsonObject componentRole = applicationRoles - .get(entry.getKey()) == null ? EMPTY_JSON_OBJECT - : applicationRoles.get(entry.getKey()).getAsJsonObject(); - JsonObject liveContainers = entry.getValue().getAsJsonObject(); - if (liveContainers != null) { - for (Map.Entry liveContainerEntry : liveContainers - .entrySet()) { - String containerId = liveContainerEntry.getKey(); - Container container = new Container(); - container.setId(containerId); - JsonObject liveContainer = (JsonObject) liveContainerEntry - .getValue(); - container - .setLaunchTime(liveContainer.get("startTime") == null ? null - : new Date(liveContainer.get("startTime").getAsLong())); - container - .setComponentName(jsonGetAsString(liveContainer, "role")); - container.setIp(jsonGetAsString(liveContainer, "ip")); - // If ip is non-null increment count - if (container.getIp() != null) { - totalNumberOfIpAssignedContainers++; - } - container.setHostname(jsonGetAsString(liveContainer, "hostname")); - container.setState(ContainerState.INIT); - if (StringUtils.isNotEmpty(container.getIp()) - && StringUtils.isNotEmpty(container.getHostname())) { - container.setState(ContainerState.READY); - } - container.setBareHost(jsonGetAsString(liveContainer, "host")); - container.setUri(CONTEXT_ROOT + APPLICATIONS_API_RESOURCE_PATH - + "/" + appName + CONTAINERS_API_RESOURCE_PATH + "/" - + containerId); - Resource resource = new Resource(); - resource.setCpus(jsonGetAsInt(componentRole, "yarn.vcores")); - resource.setMemory(jsonGetAsString(componentRole, "yarn.memory")); - container.setResource(resource); - Artifact artifact = new Artifact(); - String dockerImageName = jsonGetAsString(componentRole, - "docker.image"); - if (StringUtils.isNotEmpty(dockerImageName)) { - artifact.setId(dockerImageName); - artifact.setType(Artifact.TypeEnum.DOCKER); - } else { - // Might have to handle tarballs here - artifact.setType(null); - } - container.setArtifact(artifact); - container.setPrivilegedContainer( - jsonGetAsBoolean(componentRole, "docker.usePrivileged")); - // TODO: add container property - for response only? - app.addContainer(container); - } - } - } - } - } - - // application info - if (applicationRoles != null && !componentNames.isEmpty()) { - JsonObject applicationRole = jsonGetAsObject(applicationRoles, - componentNames.get(0)); - if (applicationRole != null) { - Artifact artifact = new Artifact(); - // how to get artifact id - docker image name?? - artifact.setId(null); - } - } - - // actual and expected number of containers - if (applicationStatistics != null) { - for (Entry entry : applicationStatistics.entrySet()) { - if (entry.getKey().equals(SLIDER_APPMASTER_COMPONENT_NAME)) { - continue; - } - JsonObject containerStats = (JsonObject) entry.getValue(); - totalNumberOfRunningContainers += jsonGetAsInt(containerStats, - "containers.live"); - totalExpectedNumberOfRunningContainers += jsonGetAsInt(containerStats, - "containers.desired"); - } - app.setNumberOfContainers(totalExpectedNumberOfRunningContainers); - app.setNumberOfRunningContainers(totalNumberOfRunningContainers); - } - - // If all containers of the app has IP assigned, then according to the REST - // API it is considered to be READY. Note, application readiness from - // end-users point of view, is out of scope of the REST API. Also, this - // readiness has nothing to do with readiness-check defined at the component - // level (which is used for dependency resolution of component DAG). - if (totalNumberOfIpAssignedContainers - .longValue() == totalExpectedNumberOfRunningContainers.longValue()) { - app.setState(ApplicationState.READY); - } - logger.info("Application = {}", app); - return Response.ok(app).build(); - } - - private String jsonGetAsString(JsonObject object, String key) { - return object.get(key) == null ? null : object.get(key).getAsString(); - } - - private Integer jsonGetAsInt(JsonObject object, String key) { - return object.get(key) == null ? null - : object.get(key).isJsonNull() ? null : object.get(key).getAsInt(); - } - - private Boolean jsonGetAsBoolean(JsonObject object, String key) { - return object.get(key) == null ? null - : object.get(key).isJsonNull() ? null : object.get(key).getAsBoolean(); - } - - private JsonObject jsonGetAsObject(JsonObject object, String key) { - return object.get(key) == null ? null : object.get(key).getAsJsonObject(); - } - - private long queryLifetime(String appName) { - try { - return invokeSliderClientRunnable( - new SliderClientContextRunnable() { - @Override - public Long run(SliderClient sliderClient) - throws YarnException, IOException, InterruptedException { - ApplicationReport report = sliderClient.findInstance(appName); - return report.getApplicationTimeouts() - .get(ApplicationTimeoutType.LIFETIME).getRemainingTime(); - } - }); - } catch (Exception e) { - logger.error("Error when querying lifetime for " + appName, e); - return DEFAULT_UNLIMITED_LIFETIME; - } - } - - private JsonObject getSliderApplicationStatus(final String appName) - throws IOException, YarnException, InterruptedException { - - return invokeSliderClientRunnable( - new SliderClientContextRunnable() { - @Override - public JsonObject run(SliderClient sliderClient) - throws YarnException, IOException, InterruptedException { - String status = null; - try { - status = sliderClient.actionStatus(appName); - } catch (BadClusterStateException e) { - logger.warn("Application not running yet", e); - return EMPTY_JSON_OBJECT; - } catch (Exception e) { - logger.error("Exception calling slider.actionStatus", e); - return EMPTY_JSON_OBJECT; - } - JsonElement statusElement = JSON_PARSER.parse(status); - return (statusElement == null || statusElement instanceof JsonNull) - ? EMPTY_JSON_OBJECT : (JsonObject) statusElement; - } - }); - } - - private JsonObject getSliderApplicationRegistry(final String appName, - final String registryName) - throws IOException, YarnException, InterruptedException { - final ActionRegistryArgs registryArgs = new ActionRegistryArgs(); - registryArgs.name = appName; - registryArgs.getConf = registryName; - registryArgs.format = ConfigFormat.JSON.toString(); - - return invokeSliderClientRunnable( - new SliderClientContextRunnable() { - @Override - public JsonObject run(SliderClient sliderClient) - throws YarnException, IOException, InterruptedException { - String registry = null; - try { - registry = sliderClient.actionRegistryGetConfig(registryArgs) - .asJson(); - } catch (FileNotFoundException | NotFoundException e) { - // ignore and return empty object - return EMPTY_JSON_OBJECT; - } catch (Exception e) { - logger.error("Exception calling slider.actionRegistryGetConfig", - e); - return EMPTY_JSON_OBJECT; - } - JsonElement registryElement = JSON_PARSER.parse(registry); - return (registryElement == null - || registryElement instanceof JsonNull) ? EMPTY_JSON_OBJECT - : (JsonObject) registryElement; - } - }); - } - - private Integer getSliderList(final String appName) - throws IOException, YarnException, InterruptedException { - return getSliderList(appName, true); - } - - private Integer getSliderList(final String appName, final boolean liveOnly) - throws IOException, YarnException, InterruptedException { - return invokeSliderClientRunnable(new SliderClientContextRunnable() { - @Override - public Integer run(SliderClient sliderClient) throws YarnException, - IOException, InterruptedException { - int status = 0; - if (liveOnly) { - status = sliderClient.actionList(appName); - } else { - status = sliderClient.actionList(appName, ACTION_LIST_ARGS); - } - return status; - } - }); - } - - private Set getSliderApplications(final String state) - throws IOException, YarnException, InterruptedException { - return getSliderApplications(false, state); - } - - private Set getSliderApplications(final boolean liveOnly) - throws IOException, YarnException, InterruptedException { - return getSliderApplications(liveOnly, null); - } - - private Set getSliderApplications(final boolean liveOnly, - final String state) - throws IOException, YarnException, InterruptedException { - return invokeSliderClientRunnable( - new SliderClientContextRunnable>() { - @Override - public Set run(SliderClient sliderClient) - throws YarnException, IOException, InterruptedException { - Set apps; - ActionListArgs listArgs = new ActionListArgs(); - if (liveOnly) { - apps = sliderClient.getApplicationList(null); - } else if (StringUtils.isNotEmpty(state)) { - listArgs.state = state; - apps = sliderClient.getApplicationList(null, listArgs); - } else { - apps = sliderClient.getApplicationList(null, listArgs); - } - return apps; - } - }); - } - @DELETE @Path("/{app_name}") @Consumes({ MediaType.APPLICATION_JSON }) @Produces({ MediaType.APPLICATION_JSON }) public Response deleteApplication(@PathParam("app_name") String appName) { logger.info("DELETE: deleteApplication for appName = {}", appName); + return stopApplication(appName, true); + } + private Response stopApplication(String appName, boolean destroy) { try { - Response stopResponse = stopSliderApplication(appName); - if (stopResponse.getStatus() == Status.INTERNAL_SERVER_ERROR - .getStatusCode()) { - return Response.status(Status.NOT_FOUND).build(); + SLIDER_CLIENT.actionStop(appName, ACTION_FREEZE_ARGS); + if (destroy) { + SLIDER_CLIENT.actionDestroy(appName); + logger.info("Successfully deleted application {}", appName); + } else { + logger.info("Successfully stopped application {}", appName); } - } catch (UnknownApplicationInstanceException e) { - logger.error("Application does not exist", e); - return Response.status(Status.NOT_FOUND).build(); + return Response.status(Status.NO_CONTENT).build(); + } catch (ApplicationNotFoundException e) { + ApplicationStatus applicationStatus = new ApplicationStatus(); + applicationStatus.setDiagnostics( + "Application " + appName + " not found " + e.getMessage()); + return Response.status(Status.NOT_FOUND).entity(applicationStatus) + .build(); } catch (Exception e) { - logger.error("Delete application failed", e); - return Response.status(Status.INTERNAL_SERVER_ERROR).build(); - } - - // Although slider client stop returns immediately, it usually takes a - // little longer for it to stop from YARN point of view. Slider destroy - // fails if the application is not completely stopped. Hence the need to - // call destroy in a controlled loop few times (only if exit code is - // EXIT_APPLICATION_IN_USE or EXIT_INSTANCE_EXISTS), before giving up. - boolean keepTrying = true; - int maxDeleteAttempts = 5; - int deleteAttempts = 0; - int sleepIntervalInMillis = 500; - while (keepTrying && deleteAttempts < maxDeleteAttempts) { - try { - destroySliderApplication(appName); - keepTrying = false; - } catch (SliderException e) { - if (e.getExitCode() == SliderExitCodes.EXIT_APPLICATION_IN_USE - || e.getExitCode() == SliderExitCodes.EXIT_INSTANCE_EXISTS) { - deleteAttempts++; - // If we used up all the allowed delete attempts, let's log it as - // error before giving up. Otherwise log as warn. - if (deleteAttempts < maxDeleteAttempts) { - logger.warn("Application not in stopped state, waiting for {}ms" - + " before trying delete again", sleepIntervalInMillis); - } else { - logger.error("Delete application failed", e); - } - try { - Thread.sleep(sleepIntervalInMillis); - } catch (InterruptedException e1) { - } - } else { - logger.error("Delete application threw exception", e); - return Response.status(Status.INTERNAL_SERVER_ERROR).build(); - } - } catch (Exception e) { - logger.error("Delete application failed", e); - return Response.status(Status.INTERNAL_SERVER_ERROR).build(); - } + ApplicationStatus applicationStatus = new ApplicationStatus(); + applicationStatus.setDiagnostics(e.getMessage()); + return Response.status(Status.INTERNAL_SERVER_ERROR) + .entity(applicationStatus).build(); } - return Response.status(Status.NO_CONTENT).build(); } - private Response stopSliderApplication(final String appName) - throws IOException, YarnException, InterruptedException { - return invokeSliderClientRunnable(new SliderClientContextRunnable() { - @Override - public Response run(SliderClient sliderClient) throws YarnException, - IOException, InterruptedException { - int returnCode = sliderClient.actionFreeze(appName, ACTION_FREEZE_ARGS); - if (returnCode == 0) { - logger.info("Successfully stopped application {}", appName); - return Response.status(Status.NO_CONTENT).build(); - } else { - logger.error("Stop of application {} failed with return code ", - appName, returnCode); - ApplicationStatus applicationStatus = new ApplicationStatus(); - applicationStatus.setDiagnostics("Stop of application " + appName - + " failed"); - return Response.status(Status.INTERNAL_SERVER_ERROR) - .entity(applicationStatus).build(); - } - } - }); - } - - private Response startSliderApplication(final String appName, Application app) - throws IOException, YarnException, InterruptedException { - return invokeSliderClientRunnable(new SliderClientContextRunnable() { - @Override - public Response run(SliderClient sliderClient) throws YarnException, - IOException, InterruptedException { - ActionThawArgs thawArgs = new ActionThawArgs(); - if (app.getLifetime() == null) { - app.setLifetime(DEFAULT_UNLIMITED_LIFETIME); - } - thawArgs.lifetime = app.getLifetime(); - int returnCode = sliderClient.actionThaw(appName, thawArgs); - if (returnCode == 0) { - logger.info("Successfully started application {}", appName); - ApplicationStatus applicationStatus = new ApplicationStatus(); - applicationStatus.setState(ApplicationState.ACCEPTED); - applicationStatus.setUri(CONTEXT_ROOT - + APPLICATIONS_API_RESOURCE_PATH + "/" + appName); - // 202 = ACCEPTED - return Response.status(HTTP_STATUS_CODE_ACCEPTED) - .entity(applicationStatus).build(); - } else { - logger.error("Start of application {} failed with returnCode ", - appName, returnCode); - ApplicationStatus applicationStatus = new ApplicationStatus(); - applicationStatus.setDiagnostics("Start of application " + appName - + " failed"); - return Response.status(Status.INTERNAL_SERVER_ERROR) - .entity(applicationStatus).build(); - } - } - }); - } + @PUT @Path("/{app_name}/{component_name}") + @Consumes({ MediaType.APPLICATION_JSON }) + @Produces({ MediaType.APPLICATION_JSON }) + public Response updateComponent(@PathParam("app_name") String appName, + @PathParam("component_name") String componentName, Component component) { - private Void destroySliderApplication(final String appName) - throws IOException, YarnException, InterruptedException { - return invokeSliderClientRunnable(new SliderClientContextRunnable() { - @Override - public Void run(SliderClient sliderClient) throws YarnException, - IOException, InterruptedException { - sliderClient.actionDestroy(appName); - return null; - } - }); + if (component.getNumberOfContainers() < 0) { + return Response.status(Status.BAD_REQUEST).entity( + "Application = " + appName + ", Component = " + component.getName() + + ": Invalid number of containers specified " + component + .getNumberOfContainers()).build(); + } + try { + long original = SLIDER_CLIENT.flex(appName, component); + return Response.ok().entity( + "Updating " + componentName + " size from " + original + " to " + + component.getNumberOfContainers()).build(); + } catch (YarnException | IOException e) { + ApplicationStatus status = new ApplicationStatus(); + status.setDiagnostics(e.getMessage()); + return Response.status(Status.INTERNAL_SERVER_ERROR).entity(status) + .build(); + } } - @PUT - @Path("/{app_name}") + @PUT @Path("/{app_name}") @Consumes({ MediaType.APPLICATION_JSON }) @Produces({ MediaType.APPLICATION_JSON }) public Response updateApplication(@PathParam("app_name") String appName, @@ -1354,158 +463,79 @@ public Response updateApplication(@PathParam("app_name") String appName, // path param updateAppData.setName(appName); - // Adding support for stop and start // For STOP the app should be running. If already stopped then this // operation will be a no-op. For START it should be in stopped state. // If already running then this operation will be a no-op. - - // Check if app exists in any state - try { - int appsFound = getSliderList(appName, false); - if (appsFound < 0) { - return Response.status(Status.NOT_FOUND).build(); - } - } catch (Exception e) { - logger.error("Update application failed", e); - return Response.status(Status.NOT_FOUND).build(); - } - - // If a STOP is requested if (updateAppData.getState() != null && updateAppData.getState() == ApplicationState.STOPPED) { - try { - int livenessCheck = getSliderList(appName); - if (livenessCheck == 0) { - return stopSliderApplication(appName); - } else { - logger.info("Application {} is already stopped", appName); - ApplicationStatus applicationStatus = new ApplicationStatus(); - applicationStatus.setDiagnostics("Application " + appName - + " is already stopped"); - return Response.status(Status.BAD_REQUEST).entity(applicationStatus) - .build(); - } - } catch (Exception e) { - logger.error("Stop application failed", e); - return Response.status(Status.INTERNAL_SERVER_ERROR).build(); - } + return stopApplication(appName, false); } // If a START is requested if (updateAppData.getState() != null && updateAppData.getState() == ApplicationState.STARTED) { - try { - int livenessCheck = getSliderList(appName); - if (livenessCheck != 0) { - return startSliderApplication(appName, updateAppData); - } else { - logger.info("Application {} is already running", appName); - ApplicationStatus applicationStatus = new ApplicationStatus(); - applicationStatus.setDiagnostics("Application " + appName - + " is already running"); - applicationStatus.setUri(CONTEXT_ROOT - + APPLICATIONS_API_RESOURCE_PATH + "/" + appName); - return Response.status(Status.BAD_REQUEST).entity(applicationStatus) - .build(); - } - } catch (Exception e) { - logger.error("Start application failed", e); - return Response.status(Status.INTERNAL_SERVER_ERROR).build(); - } - } - - // If no of instances specified then treat it as a flex - if (updateAppData.getNumberOfContainers() != null - && updateAppData.getComponents() == null) { - updateAppData.setComponents(getDefaultComponentAsList()); - } - - // At this point if there are components then it is a flex - if (updateAppData.getComponents() != null) { - try { - int livenessCheck = getSliderList(appName); - if (livenessCheck == 0) { - flexSliderApplication(appName, updateAppData); - } - return Response.status(Status.NO_CONTENT).build(); - } catch (Exception e) { - logger.error("Update application failed", e); - return Response.status(Status.INTERNAL_SERVER_ERROR).build(); - } + return startApplication(appName); } // If new lifetime value specified then update it if (updateAppData.getLifetime() != null && updateAppData.getLifetime() > 0) { - try { - updateAppLifetime(appName, updateAppData.getLifetime()); - } catch (Exception e) { - logger.error("Failed to update application (" + appName + ") lifetime (" - + updateAppData.getLifetime() + ")", e); - return Response.status(Status.INTERNAL_SERVER_ERROR).build(); - } + return updateLifetime(appName, updateAppData); + } + + // flex a single component app + if (updateAppData.getNumberOfContainers() != null + && updateAppData.getComponents() == null) { + Component defaultComp = createDefaultComponent(updateAppData); + return updateComponent(updateAppData.getName(), defaultComp.getName(), + defaultComp); } // If nothing happens consider it a no-op return Response.status(Status.NO_CONTENT).build(); } - private Void updateAppLifetime(String appName, long lifetime) - throws InterruptedException, YarnException, IOException { - return invokeSliderClientRunnable(new SliderClientContextRunnable() { - @Override public Void run(SliderClient sliderClient) - throws YarnException, IOException, InterruptedException { - ActionUpdateArgs args = new ActionUpdateArgs(); - args.lifetime = lifetime; - sliderClient.actionUpdate(appName, args); - return null; - } - }); + private Response updateLifetime(String appName, Application updateAppData) { + try { + String newLifeTime = + SLIDER_CLIENT.updateLifetime(appName, updateAppData.getLifetime()); + return Response.ok("Application " + appName + " lifeTime is successfully updated to " + + updateAppData.getLifetime() + " seconds from now: " + newLifeTime).build(); + } catch (Exception e) { + String message = + "Failed to update application (" + appName + ") lifetime (" + + updateAppData.getLifetime() + ")"; + logger.error(message, e); + return Response.status(Status.INTERNAL_SERVER_ERROR) + .entity(message + " : " + e.getMessage()).build(); + } } - // create default component and initialize with app level global values - private List getDefaultComponentAsList(Application app) { - List comps = getDefaultComponentAsList(); - Component comp = comps.get(0); - comp.setArtifact(app.getArtifact()); - comp.setResource(app.getResource()); - comp.setNumberOfContainers(app.getNumberOfContainers()); - comp.setLaunchCommand(app.getLaunchCommand()); - return comps; + private Response startApplication(String appName) { + try { + int ret = SLIDER_CLIENT.actionList(appName); + if (ret == 0) { + return Response.ok() + .entity("Application " + appName + " is already alive.").build(); + } + SLIDER_CLIENT.actionThaw(appName, null); + logger.info("Successfully started application " + appName); + return Response.ok("Application " + appName + " is successfully started").build(); + } catch (Exception e) { + String message = "Failed to start application " + appName; + logger.info(message, e); + return Response.status(Status.INTERNAL_SERVER_ERROR) + .entity(message + ": " + e.getMessage()).build(); + } } - private List getDefaultComponentAsList() { + private Component createDefaultComponent(Application app) { Component comp = new Component(); comp.setName(DEFAULT_COMPONENT_NAME); - List comps = new ArrayList<>(); - comps.add(comp); - return comps; - } - - private Void flexSliderApplication(final String appName, - final Application updateAppData) throws IOException, YarnException, - InterruptedException { - return invokeSliderClientRunnable(new SliderClientContextRunnable() { - @Override - public Void run(SliderClient sliderClient) throws YarnException, - IOException, InterruptedException { - ActionFlexArgs flexArgs = new ActionFlexArgs(); - ComponentArgsDelegate compDelegate = new ComponentArgsDelegate(); - Long globalNumberOfContainers = updateAppData.getNumberOfContainers(); - for (Component comp : updateAppData.getComponents()) { - Long noOfContainers = comp.getNumberOfContainers() == null - ? globalNumberOfContainers : comp.getNumberOfContainers(); - if (noOfContainers != null) { - compDelegate.componentTuples.addAll( - Arrays.asList(comp.getName(), String.valueOf(noOfContainers))); - } - } - if (!compDelegate.componentTuples.isEmpty()) { - flexArgs.componentDelegate = compDelegate; - sliderClient.actionFlex(appName, flexArgs); - } - return null; - } - }); + comp.setArtifact(app.getArtifact()); + comp.setResource(app.getResource()); + comp.setNumberOfContainers(app.getNumberOfContainers()); + comp.setLaunchCommand(app.getLaunchCommand()); + return comp; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/services/api/impl/TestApplicationApiService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/services/api/impl/TestApplicationApiService.java index 7bfb410..984044a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/services/api/impl/TestApplicationApiService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/services/api/impl/TestApplicationApiService.java @@ -20,6 +20,7 @@ import static org.apache.hadoop.yarn.services.utils.RestApiConstants.*; import static org.apache.hadoop.yarn.services.utils.RestApiErrorMessages.*; +import java.util.ArrayList; import java.util.HashMap; import java.util.Map; @@ -61,12 +62,10 @@ public void tearDown() throws Exception { @Test(timeout = 90000) public void testValidateApplicationPostPayload() throws Exception { Application app = new Application(); - Map compNameArtifactIdMap = new HashMap<>(); // no name try { - appApiService.validateApplicationPostPayload(app, - compNameArtifactIdMap); + appApiService.validateApplicationPostPayload(app); Assert.fail(EXCEPTION_PREFIX + "application with no name"); } catch (IllegalArgumentException e) { Assert.assertEquals(ERROR_APPLICATION_NAME_INVALID, e.getMessage()); @@ -77,8 +76,7 @@ public void testValidateApplicationPostPayload() throws Exception { for (String badName : badNames) { app.setName(badName); try { - appApiService.validateApplicationPostPayload(app, - compNameArtifactIdMap); + appApiService.validateApplicationPostPayload(app); Assert.fail(EXCEPTION_PREFIX + "application with bad name " + badName); } catch (IllegalArgumentException e) { Assert.assertEquals(ERROR_APPLICATION_NAME_INVALID_FORMAT, @@ -89,8 +87,7 @@ public void testValidateApplicationPostPayload() throws Exception { // no artifact app.setName("finance_home"); try { - appApiService.validateApplicationPostPayload(app, - compNameArtifactIdMap); + appApiService.validateApplicationPostPayload(app); Assert.fail(EXCEPTION_PREFIX + "application with no artifact"); } catch (IllegalArgumentException e) { Assert.assertEquals(ERROR_ARTIFACT_INVALID, e.getMessage()); @@ -100,8 +97,7 @@ public void testValidateApplicationPostPayload() throws Exception { Artifact artifact = new Artifact(); app.setArtifact(artifact); try { - appApiService.validateApplicationPostPayload(app, - compNameArtifactIdMap); + appApiService.validateApplicationPostPayload(app); Assert.fail(EXCEPTION_PREFIX + "application with no artifact id"); } catch (IllegalArgumentException e) { Assert.assertEquals(ERROR_ARTIFACT_ID_INVALID, e.getMessage()); @@ -112,8 +108,7 @@ public void testValidateApplicationPostPayload() throws Exception { artifact.setId("app.io/hbase:facebook_0.2"); app.setNumberOfContainers(5l); try { - appApiService.validateApplicationPostPayload(app, - compNameArtifactIdMap); + appApiService.validateApplicationPostPayload(app); } catch (IllegalArgumentException e) { logger.error("application attributes specified should be valid here", e); Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage()); @@ -124,22 +119,18 @@ public void testValidateApplicationPostPayload() throws Exception { Assert.assertEquals(app.getComponents().get(0).getName(), DEFAULT_COMPONENT_NAME); Assert.assertEquals(app.getLifetime(), DEFAULT_UNLIMITED_LIFETIME); - Assert.assertEquals("Property not set", - app.getConfiguration().getProperties() - .get(SliderKeys.COMPONENT_TYPE_KEY), - SliderKeys.COMPONENT_TYPE_EXTERNAL_APP); + //TODO handle external app // unset artifact type, default component and no of containers to test other // validation logic artifact.setType(null); - app.setComponents(null); + app.setComponents(new ArrayList<>()); app.setNumberOfContainers(null); // resource not specified artifact.setId("docker.io/centos:centos7"); try { - appApiService.validateApplicationPostPayload(app, - compNameArtifactIdMap); + appApiService.validateApplicationPostPayload(app); Assert.fail(EXCEPTION_PREFIX + "application with no resource"); } catch (IllegalArgumentException e) { Assert.assertEquals(ERROR_RESOURCE_INVALID, e.getMessage()); @@ -149,28 +140,18 @@ public void testValidateApplicationPostPayload() throws Exception { Resource res = new Resource(); app.setResource(res); try { - appApiService.validateApplicationPostPayload(app, - compNameArtifactIdMap); + appApiService.validateApplicationPostPayload(app); Assert.fail(EXCEPTION_PREFIX + "application with no memory"); } catch (IllegalArgumentException e) { Assert.assertEquals(ERROR_RESOURCE_MEMORY_INVALID, e.getMessage()); } - // cpus not specified - res.setMemory("2gb"); - try { - appApiService.validateApplicationPostPayload(app, - compNameArtifactIdMap); - Assert.fail(EXCEPTION_PREFIX + "application with no cpu"); - } catch (IllegalArgumentException e) { - Assert.assertEquals(ERROR_RESOURCE_CPUS_INVALID, e.getMessage()); - } - + // cpu does not need to be always specified, it's an optional feature in yarn // invalid no of cpus + res.setMemory("100mb"); res.setCpus(-2); try { - appApiService.validateApplicationPostPayload(app, - compNameArtifactIdMap); + appApiService.validateApplicationPostPayload(app); Assert.fail( EXCEPTION_PREFIX + "application with invalid no of cpups"); } catch (IllegalArgumentException e) { @@ -180,8 +161,7 @@ public void testValidateApplicationPostPayload() throws Exception { // number of containers not specified res.setCpus(2); try { - appApiService.validateApplicationPostPayload(app, - compNameArtifactIdMap); + appApiService.validateApplicationPostPayload(app); Assert.fail( EXCEPTION_PREFIX + "application with no container count"); } catch (IllegalArgumentException e) { @@ -191,8 +171,7 @@ public void testValidateApplicationPostPayload() throws Exception { // specifying profile along with cpus/memory raises exception res.setProfile("hbase_finance_large"); try { - appApiService.validateApplicationPostPayload(app, - compNameArtifactIdMap); + appApiService.validateApplicationPostPayload(app); Assert.fail(EXCEPTION_PREFIX + "application with resource profile along with cpus/memory"); } catch (IllegalArgumentException e) { @@ -205,8 +184,7 @@ public void testValidateApplicationPostPayload() throws Exception { res.setCpus(null); res.setMemory(null); try { - appApiService.validateApplicationPostPayload(app, - compNameArtifactIdMap); + appApiService.validateApplicationPostPayload(app); Assert.fail(EXCEPTION_PREFIX + "application with resource profile only - NOT SUPPORTED"); } catch (IllegalArgumentException e) { @@ -222,8 +200,7 @@ public void testValidateApplicationPostPayload() throws Exception { // everything valid here app.setNumberOfContainers(5l); try { - appApiService.validateApplicationPostPayload(app, - compNameArtifactIdMap); + appApiService.validateApplicationPostPayload(app); } catch (IllegalArgumentException e) { logger.error("application attributes specified should be valid here", e); Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderApplicationApi.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderApplicationApi.java index d21785f..f6a2cc9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderApplicationApi.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderApplicationApi.java @@ -60,14 +60,6 @@ ConfTreeOperations getDesiredResources() throws IOException; /** - * Put an updated resources structure. This triggers a cluster flex - * operation - * @param updated updated resources - * @throws IOException on any problem. - */ - void putDesiredResources(ConfTree updated) throws IOException; - - /** * Get the aggregate resolved model * @return the aggregate configuration of what was asked for * -after resolution has taken place diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java index 893e706..f384927 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java @@ -52,12 +52,9 @@ Messages.UpgradeContainersRequestProto request) throws IOException, YarnException; - /** - * Flex the cluster. - */ - Messages.FlexClusterResponseProto flexCluster(Messages.FlexClusterRequestProto request) - throws IOException; + Messages.FlexComponentResponseProto flexComponent( + Messages.FlexComponentRequestProto request) throws IOException; /** * Get the current cluster status @@ -121,13 +118,6 @@ throws IOException; /** - * Get the instance definition - */ - Messages.GetInstanceDefinitionResponseProto getInstanceDefinition( - Messages.GetInstanceDefinitionRequestProto request) - throws IOException, YarnException; - - /** * Get the application liveness * @return current liveness information * @throws IOException diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Application.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Application.java index cc3355a..4925e91 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Application.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Application.java @@ -28,6 +28,7 @@ import javax.xml.bind.annotation.XmlRootElement; import java.util.ArrayList; import java.util.Date; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -55,11 +56,11 @@ private Long numberOfRunningContainers = null; private Long lifetime = null; private PlacementPolicy placementPolicy = null; - private List components = null; - private Configuration configuration = null; + private List components = new ArrayList<>(); + private Configuration configuration = new Configuration(); private List containers = new ArrayList<>(); private ApplicationState state = null; - private Map quicklinks = null; + private Map quicklinks = new HashMap<>(); private String queue = null; /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Component.java index 4f50564..486b08b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Component.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Component.java @@ -49,14 +49,14 @@ private String name = null; private List dependencies = new ArrayList(); private ReadinessCheck readinessCheck = null; - private Artifact artifact = null; + private Artifact artifact = new Artifact(); private String launchCommand = null; - private Resource resource = null; + private Resource resource = new Resource(); private Long numberOfContainers = null; - private Boolean uniqueComponentSupport = null; - private Boolean runPrivilegedContainer = null; + private Boolean uniqueComponentSupport = false; + private Boolean runPrivilegedContainer = false; private PlacementPolicy placementPolicy = null; - private Configuration configuration = null; + private Configuration configuration = new Configuration(); private List quicklinks = new ArrayList(); /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/ConfigFile.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/ConfigFile.java index bad68c1..cdc96b8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/ConfigFile.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/ConfigFile.java @@ -21,6 +21,7 @@ import io.swagger.annotations.ApiModelProperty; import java.io.Serializable; +import java.util.Map; import java.util.Objects; import javax.xml.bind.annotation.XmlElement; @@ -62,7 +63,7 @@ public String toString() { private TypeEnum type = null; private String destFile = null; private String srcFile = null; - private Object props = null; + private Map props = null; /** * Config file in the standard format like xml, properties, json, yaml, @@ -104,6 +105,8 @@ public void setDestFile(String destFile) { } /** + * TODO this probably is not required for non-template configs. It is now used as symlink for localization for non-template configs - we could infer the name from destFile instead + * * Required for type template. This provides the source location of the * template which needs to be mounted as dest_file post property * substitutions. Typically the src_file would point to a source controlled @@ -131,21 +134,36 @@ public void setSrcFile(String srcFile) { * src_file is mandatory and the src_file content is dumped to dest_file post * property substitutions. **/ - public ConfigFile props(Object props) { + public ConfigFile props(Map props) { this.props = props; return this; } @ApiModelProperty(example = "null", value = "A blob of key value pairs that will be dumped in the dest_file in the format as specified in type. If the type is template then the attribute src_file is mandatory and the src_file content is dumped to dest_file post property substitutions.") @JsonProperty("props") - public Object getProps() { + public Map getProps() { return props; } - public void setProps(Object props) { + public void setProps(Map props) { this.props = props; } + public long getLong(String name, long defaultValue) { + if (name == null) { + return defaultValue; + } + String value = props.get(name.trim()); + return Long.parseLong(value); + } + + public boolean getBoolean(String name, boolean defaultValue) { + if (name == null) { + return defaultValue; + } + return Boolean.valueOf(props.get(name.trim())); + } + @Override public boolean equals(java.lang.Object o) { if (this == o) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java index c4f2ad4..c43bd64 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java @@ -29,6 +29,7 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.commons.lang.StringUtils; /** * Set of configuration properties that can be injected into the application @@ -104,6 +105,35 @@ public void setFiles(List files) { this.files = files; } + public long getPropertyLong(String name, long defaultValue) { + if (name == null) { + return defaultValue; + } + String value = properties.get(name.trim()); + if (StringUtils.isEmpty(value)) { + return defaultValue; + } + return Long.parseLong(value); + } + + public String getProperty(String name, String defaultValue) { + if (name == null) { + return defaultValue; + } + return properties.get(name.trim()); + } + + public void setProperty(String name, String value) { + properties.put(name, value); + } + + public String getProperty(String name) { + if (name == null) { + return null; + } + return properties.get(name.trim()); + } + @Override public boolean equals(java.lang.Object o) { if (this == o) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Resource.java index 190121d..c255369 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Resource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Resource.java @@ -39,7 +39,7 @@ private static final long serialVersionUID = -6431667797380250037L; private String profile = null; - private Integer cpus = null; + private Integer cpus = 1; private String memory = null; /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java index 00e2b62..03c22c9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -55,18 +55,24 @@ import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ApplicationTimeout; import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType; +import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.LocalResource; +import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.NodeState; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.client.api.YarnClientApplication; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException; import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.util.ConverterUtils; +import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.util.Times; import org.apache.slider.api.ClusterDescription; import org.apache.slider.api.ClusterNode; @@ -74,6 +80,8 @@ import org.apache.slider.api.SliderClusterProtocol; import org.apache.slider.api.StateValues; import org.apache.slider.api.proto.Messages; +import org.apache.slider.api.resource.Application; +import org.apache.slider.api.resource.Component; import org.apache.slider.api.types.ContainerInformation; import org.apache.slider.api.types.NodeInformationList; import org.apache.slider.api.types.SliderInstanceDescription; @@ -82,11 +90,11 @@ import org.apache.slider.common.Constants; import org.apache.slider.common.SliderExitCodes; import org.apache.slider.common.SliderKeys; +import org.apache.slider.common.SliderXmlConfKeys; import org.apache.slider.common.params.AbstractActionArgs; import org.apache.slider.common.params.AbstractClusterBuildingActionArgs; import org.apache.slider.common.params.ActionAMSuicideArgs; import org.apache.slider.common.params.ActionClientArgs; -import org.apache.slider.common.params.ActionCreateArgs; import org.apache.slider.common.params.ActionDependencyArgs; import org.apache.slider.common.params.ActionDestroyArgs; import org.apache.slider.common.params.ActionDiagnosticArgs; @@ -113,20 +121,16 @@ import org.apache.slider.common.params.Arguments; import org.apache.slider.common.params.ClientArgs; import org.apache.slider.common.params.CommonArgs; -import org.apache.slider.common.params.LaunchArgsAccessor; import org.apache.slider.common.tools.ConfigHelper; import org.apache.slider.common.tools.Duration; import org.apache.slider.common.tools.SliderFileSystem; import org.apache.slider.common.tools.SliderUtils; import org.apache.slider.common.tools.SliderVersionInfo; -import org.apache.slider.core.buildutils.InstanceBuilder; import org.apache.slider.core.buildutils.InstanceIO; import org.apache.slider.core.conf.AggregateConf; import org.apache.slider.core.conf.ConfTree; import org.apache.slider.core.conf.ConfTreeOperations; import org.apache.slider.core.conf.MapOperations; -import org.apache.slider.core.conf.ResourcesInputPropertiesValidator; -import org.apache.slider.core.conf.TemplateInputPropertiesValidator; import org.apache.slider.core.exceptions.BadClusterStateException; import org.apache.slider.core.exceptions.BadCommandArgumentsException; import org.apache.slider.core.exceptions.BadConfigException; @@ -137,16 +141,12 @@ import org.apache.slider.core.exceptions.UnknownApplicationInstanceException; import org.apache.slider.core.exceptions.UsageException; import org.apache.slider.core.exceptions.WaitTimeoutException; -import org.apache.slider.core.launch.AppMasterLauncher; import org.apache.slider.core.launch.ClasspathConstructor; import org.apache.slider.core.launch.CredentialUtils; import org.apache.slider.core.launch.JavaCommandLineBuilder; -import org.apache.slider.core.launch.LaunchedApplication; import org.apache.slider.core.launch.SerializedApplicationReport; import org.apache.slider.core.main.RunService; -import org.apache.slider.core.persist.AppDefinitionPersister; import org.apache.slider.core.persist.ApplicationReportSerDeser; -import org.apache.slider.core.persist.ConfPersister; import org.apache.slider.core.persist.JsonSerDeser; import org.apache.slider.core.persist.LockAcquireFailedException; import org.apache.slider.core.registry.SliderRegistryUtils; @@ -160,12 +160,10 @@ import org.apache.slider.core.registry.retrieve.RegistryRetriever; import org.apache.slider.core.zk.BlockingZKWatcher; import org.apache.slider.core.zk.ZKIntegration; -import org.apache.slider.core.zk.ZKPathBuilder; import org.apache.slider.providers.AbstractClientProvider; +import org.apache.slider.providers.ProviderUtils; import org.apache.slider.providers.SliderProviderFactory; import org.apache.slider.providers.agent.AgentKeys; -import org.apache.slider.providers.docker.DockerClientProvider; -import org.apache.slider.providers.slideram.SliderAMClientProvider; import org.apache.slider.server.appmaster.SliderAppMaster; import org.apache.slider.server.appmaster.rpc.RpcBinder; import org.apache.slider.server.services.utility.AbstractSliderLaunchedService; @@ -182,7 +180,6 @@ import java.io.Console; import java.io.File; import java.io.FileNotFoundException; -import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.InterruptedIOException; @@ -194,7 +191,6 @@ import java.net.InetSocketAddress; import java.net.URISyntaxException; import java.nio.charset.Charset; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -213,14 +209,13 @@ import java.util.regex.Pattern; import static org.apache.hadoop.registry.client.binding.RegistryUtils.*; -import static org.apache.slider.api.InternalKeys.*; -import static org.apache.slider.api.OptionKeys.*; -import static org.apache.slider.api.ResourceKeys.*; +import static org.apache.slider.api.InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH; +import static org.apache.slider.api.InternalKeys.INTERNAL_PROVIDER_NAME; +import static org.apache.slider.api.ResourceKeys.COMPONENT_INSTANCES; import static org.apache.slider.common.Constants.HADOOP_JAAS_DEBUG; import static org.apache.slider.common.params.SliderActions.*; import static org.apache.slider.common.tools.SliderUtils.*; - /** * Client service for Slider */ @@ -246,6 +241,8 @@ public static final String E_NO_RESOURCE_MANAGER = "No valid Resource Manager address provided"; public static final String E_PACKAGE_EXISTS = "Package exists"; private static PrintStream clientOutputStream = System.out; + private static final JsonSerDeser jsonSerDeser = + new JsonSerDeser(Application.class); // value should not be changed without updating string find in slider.py private static final String PASSWORD_PROMPT = "Enter password for"; @@ -371,7 +368,7 @@ public int exec() throws Throwable { break; case ACTION_CREATE: - exitCode = actionCreate(clusterName, serviceArgs.getActionCreateArgs()); +// exitCode = actionCreate(clusterName, serviceArgs.getActionCreateArgs()); break; case ACTION_DEPENDENCY: @@ -379,7 +376,7 @@ public int exec() throws Throwable { break; case ACTION_DESTROY: - exitCode = actionDestroy(clusterName, serviceArgs.getActionDestroyArgs()); + actionDestroy(clusterName); break; case ACTION_DIAGNOSTICS: @@ -396,7 +393,7 @@ public int exec() throws Throwable { break; case ACTION_FREEZE: - exitCode = actionFreeze(clusterName, serviceArgs.getActionFreezeArgs()); + actionStop(clusterName, serviceArgs.getActionFreezeArgs()); break; case ACTION_HELP: @@ -516,7 +513,6 @@ public boolean deleteZookeeperNode(String clusterName) throws YarnException, IOE String zkPath = ZKIntegration.mkClusterPath(user, clusterName); Exception e = null; try { - Configuration config = getConfig(); ZKIntegration client = getZkClient(clusterName, user); if (client != null) { if (client.exists(zkPath)) { @@ -627,76 +623,31 @@ protected ZKIntegration getZkClient(String clusterName, String user) throws Yarn * force=true by default. */ @Override - public int actionDestroy(String clustername) throws YarnException, - IOException { - ActionDestroyArgs destroyArgs = new ActionDestroyArgs(); - destroyArgs.force = true; - return actionDestroy(clustername, destroyArgs); - } - - @Override - public int actionDestroy(String clustername, - ActionDestroyArgs destroyArgs) throws YarnException, IOException { - // verify that a live cluster isn't there - validateClusterName(clustername); - //no=op, it is now mandatory. - verifyBindingsDefined(); - verifyNoLiveClusters(clustername, "Destroy"); - boolean forceDestroy = destroyArgs.force; - log.debug("actionDestroy({}, force={})", clustername, forceDestroy); - - // create the directory path - Path clusterDirectory = sliderFileSystem.buildClusterDirPath(clustername); - // delete the directory; + public void actionDestroy(String appName) + throws YarnException, IOException { + validateClusterName(appName); + Path appDir = sliderFileSystem.buildClusterDirPath(appName); FileSystem fs = sliderFileSystem.getFileSystem(); - boolean exists = fs.exists(clusterDirectory); - if (exists) { - log.debug("Application Instance {} found at {}: destroying", clustername, clusterDirectory); - if (!forceDestroy) { - // fail the command if --force is not explicitly specified - throw new UsageException("Destroy will permanently delete directories and registries. " - + "Reissue this command with the --force option if you want to proceed."); - } - if (!fs.delete(clusterDirectory, true)) { - log.warn("Filesystem returned false from delete() operation"); - } - - if(!deleteZookeeperNode(clustername)) { - log.warn("Unable to perform node cleanup in Zookeeper."); - } - - if (fs.exists(clusterDirectory)) { - log.warn("Failed to delete {}", clusterDirectory); + if (fs.exists(appDir)) { + if (fs.delete(appDir, true)) { + log.info("Successfully deleted application + " + appName); + return; + } else { + String message = + "Failed to delete application + " + appName + " at: " + appDir; + log.info(message); + throw new YarnException(message); } - - } else { - log.debug("Application Instance {} already destroyed", clustername); } - - // rm the registry entry —do not let this block the destroy operations - String registryPath = SliderRegistryUtils.registryPathForInstance( - clustername); - try { - getRegistryOperations().delete(registryPath, true); - } catch (IOException e) { - log.warn("Error deleting registry entry {}: {} ", registryPath, e, e); - } catch (SliderException e) { - log.warn("Error binding to registry {} ", e, e); - } - - List instances = findAllLiveInstances(clustername); - // detect any race leading to cluster creation during the check/destroy process - // and report a problem. - if (!instances.isEmpty()) { - throw new SliderException(EXIT_APPLICATION_IN_USE, - clustername + ": " - + E_DESTROY_CREATE_RACE_CONDITION - + " :" + - instances.get(0)); + if (!deleteZookeeperNode(appName)) { + String message = + "Failed to cleanup cleanup application " + appName + " in zookeeper"; + log.warn(message); + throw new YarnException(message); } - log.info("Destroyed cluster {}", clustername); - return EXIT_SUCCESS; + //TODO clean registry } + @Override public int actionAmSuicide(String clustername, @@ -715,30 +666,268 @@ public AbstractClientProvider createClientProvider(String provider) return factory.createClientProvider(); } - /** - * Create the cluster -saving the arguments to a specification file first - * @param clustername cluster name - * @return the status code - * @throws YarnException Yarn problems - * @throws IOException other problems - * @throws BadCommandArgumentsException bad arguments. - */ - public int actionCreate(String clustername, ActionCreateArgs createArgs) throws - YarnException, - IOException { - - actionBuild(clustername, createArgs); - Path clusterDirectory = sliderFileSystem.buildClusterDirPath(clustername); - AggregateConf instanceDefinition = loadInstanceDefinitionUnresolved( - clustername, clusterDirectory); - try { - checkForCredentials(getConfig(), instanceDefinition.getAppConf(), - clustername); - } catch (IOException e) { - sliderFileSystem.getFileSystem().delete(clusterDirectory, true); - throw e; + + public ApplicationId actionCreate(Application application) + throws IOException, YarnException { + String appName = application.getName(); + validateClusterName(appName); + verifyNoLiveClusters(appName, "Create"); + + // write app definition on to hdfs + persistApp(application); + return submitApp(application); + + //TODO deal with registry + } + + private ApplicationId submitApp(Application app) + throws IOException, YarnException { + String appName = app.getName(); + Configuration conf = getConfig(); + Path appRootDir = sliderFileSystem.buildClusterDirPath(app.getName()); + deployedClusterName = appName; + + YarnClientApplication yarnApp = yarnClient.createApplication(); + ApplicationSubmissionContext submissionContext = + yarnApp.getApplicationSubmissionContext(); + applicationId = submissionContext.getApplicationId(); + submissionContext.setKeepContainersAcrossApplicationAttempts(true); + if (app.getLifetime() > 0) { + Map appTimeout = new HashMap<>(); + appTimeout.put(ApplicationTimeoutType.LIFETIME, app.getLifetime()); + submissionContext.setApplicationTimeouts(appTimeout); + } + submissionContext.setMaxAppAttempts(conf.getInt(KEY_AM_RESTART_LIMIT, 2)); + + Map localResources = + new HashMap(); + + // copy local slideram-log4j.properties to hdfs and add to localResources + boolean hasSliderAMLog4j = + addAMLog4jResource(appName, conf, localResources); + // copy jars to hdfs and add to localResources + Path tempPath = addJarResource(appName, submissionContext, localResources); + // add keytab if in secure env + addKeytabResourceIfSecure(sliderFileSystem, localResources, conf, appName); + printLocalResources(localResources); + + //TODO SliderAMClientProvider#copyEnvVars + //TODO localResource putEnv + + Map env = addAMEnv(conf, tempPath); + + // create AM CLI + String cmdStr = + buildCommandLine(appName, conf, appRootDir, hasSliderAMLog4j); + + //TODO set log aggregation context + //TODO set retry window + submissionContext.setResource(Resource.newInstance( + conf.getLong(KEY_AM_RESOURCE_MEM, DEFAULT_KEY_AM_RESOURCE_MEM), 1)); + submissionContext.setQueue(conf.get(KEY_YARN_QUEUE, DEFAULT_YARN_QUEUE)); + submissionContext.setApplicationName(appName); + submissionContext.setApplicationType(SliderKeys.APP_TYPE); + Set appTags = + AbstractClientProvider.createApplicationTags(appName, null, null); + if (!appTags.isEmpty()) { + submissionContext.setApplicationTags(appTags); + } + ContainerLaunchContext amLaunchContext = + Records.newRecord(ContainerLaunchContext.class); + amLaunchContext.setCommands(Collections.singletonList(cmdStr)); + amLaunchContext.setEnvironment(env); + amLaunchContext.setLocalResources(localResources); + addCredentialsIfSecure(conf, amLaunchContext); + submissionContext.setAMContainerSpec(amLaunchContext); + yarnClient.submitApplication(submissionContext); + return submissionContext.getApplicationId(); + } + + private void printLocalResources(Map map) { + log.info("Added LocalResource for localization: "); + StringBuilder builder = new StringBuilder(); + for (Map.Entry entry : map.entrySet()) { + builder.append(entry.getKey()).append(" -> ") + .append(entry.getValue().getResource().getFile()) + .append(System.lineSeparator()); + } + log.info(builder.toString()); + } + + private void addCredentialsIfSecure(Configuration conf, + ContainerLaunchContext amLaunchContext) throws IOException { + if (UserGroupInformation.isSecurityEnabled()) { + // pick up oozie credentials + Credentials credentials = + CredentialUtils.loadTokensFromEnvironment(System.getenv(), conf); + if (credentials == null) { + // nothing from oozie, so build up directly + credentials = new Credentials( + UserGroupInformation.getCurrentUser().getCredentials()); + CredentialUtils.addRMRenewableFSDelegationTokens(conf, + sliderFileSystem.getFileSystem(), credentials); + } else { + log.info("Using externally supplied credentials to launch AM"); + } + amLaunchContext.setTokens(CredentialUtils.marshallCredentials(credentials)); + } + } + + private String buildCommandLine(String appName, Configuration conf, + Path appRootDir, boolean hasSliderAMLog4j) throws BadConfigException { + JavaCommandLineBuilder CLI = new JavaCommandLineBuilder(); + CLI.forceIPv4().headless(); + //TODO CLI.setJVMHeap + //TODO CLI.addJVMOPTS + if (hasSliderAMLog4j) { + CLI.sysprop(SYSPROP_LOG4J_CONFIGURATION, LOG4J_SERVER_PROP_FILENAME); + CLI.sysprop(SYSPROP_LOG_DIR, ApplicationConstants.LOG_DIR_EXPANSION_VAR); + } + CLI.add(SliderAppMaster.SERVICE_CLASSNAME); + CLI.add(ACTION_CREATE, appName); + //TODO debugAM CLI.add(Arguments.ARG_DEBUG) + CLI.add(Arguments.ARG_CLUSTER_URI, appRootDir.toUri()); + InetSocketAddress rmSchedulerAddress = getRmSchedulerAddress(conf); + String rmAddr = NetUtils.getHostPortString(rmSchedulerAddress); + CLI.add(Arguments.ARG_RM_ADDR, rmAddr); + // pass the registry binding + CLI.addConfOptionToCLI(conf, RegistryConstants.KEY_REGISTRY_ZK_ROOT, + RegistryConstants.DEFAULT_ZK_REGISTRY_ROOT); + CLI.addMandatoryConfOption(conf, RegistryConstants.KEY_REGISTRY_ZK_QUORUM); + if(isHadoopClusterSecure(conf)) { + //TODO Is this required ?? + // if the cluster is secure, make sure that + // the relevant security settings go over + CLI.addConfOption(conf, DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY); + } + // copy over any/all YARN RM client values, in case the server-side XML conf file + // has the 0.0.0.0 address + CLI.addConfOptions(conf, YarnConfiguration.RM_ADDRESS, + YarnConfiguration.RM_CLUSTER_ID, YarnConfiguration.RM_HOSTNAME, + YarnConfiguration.RM_PRINCIPAL); + + // write out the path output + CLI.addOutAndErrFiles(STDOUT_AM, STDERR_AM); + String cmdStr = CLI.build(); + log.info("Completed setting up app master command: {}", cmdStr); + return cmdStr; + } + + private Map addAMEnv(Configuration conf, Path tempPath) + throws IOException { + Map env = new HashMap(); + ClasspathConstructor classpath = + buildClasspath(SliderKeys.SUBMITTED_CONF_DIR, "lib", + sliderFileSystem, getUsingMiniMRCluster()); + env.put("CLASSPATH", classpath.buildClasspath()); + env.put("LANG", "en_US.UTF-8"); + env.put("LC_ALL", "en_US.UTF-8"); + env.put("LANGUAGE", "en_US.UTF-8"); + String jaas = System.getenv(HADOOP_JAAS_DEBUG); + if (jaas != null) { + env.put(HADOOP_JAAS_DEBUG, jaas); + } + env.putAll(getAmLaunchEnv(conf)); + log.info("AM env: \n{}", stringifyMap(env)); + return env; + } + + private Path addJarResource(String appName, + ApplicationSubmissionContext submissionContext, + Map localResources) + throws IOException, SliderException { + Path tempPath = sliderFileSystem.createAppInstanceTempPath(appName, + submissionContext.getApplicationId() + "/am"); + ProviderUtils + .addProviderJar(localResources, this, SLIDER_JAR, sliderFileSystem, + tempPath, "lib", false); + log.info("Loading all dependencies for AM."); + Path dependencyLibTarGzip = sliderFileSystem.getDependencyTarGzip(); + if (sliderFileSystem.isFile(dependencyLibTarGzip)) { + SliderUtils.putAmTarGzipAndUpdate(localResources, sliderFileSystem); + } else { + for (String libDirProp : SliderUtils.getLibDirs()) { + ProviderUtils + .addAllDependencyJars(localResources, sliderFileSystem, tempPath, + "lib", libDirProp); + } + } + return tempPath; + } + + private boolean addAMLog4jResource(String appName, Configuration conf, + Map localResources) + throws IOException, BadClusterStateException { + boolean hasSliderAMLog4j = false; + String hadoopConfDir = + System.getenv(ApplicationConstants.Environment.HADOOP_CONF_DIR.name()); + if (hadoopConfDir != null) { + File localFile = + new File(hadoopConfDir, SliderKeys.LOG4J_SERVER_PROP_FILENAME); + if (localFile.exists()) { + Path localFilePath = createLocalPath(localFile); + Path appDirPath = sliderFileSystem.buildClusterDirPath(appName); + Path remoteConfPath = + new Path(appDirPath, SliderKeys.SUBMITTED_CONF_DIR); + Path remoteFilePath = + new Path(remoteConfPath, SliderKeys.LOG4J_SERVER_PROP_FILENAME); + copy(conf, localFilePath, remoteFilePath); + LocalResource localResource = sliderFileSystem + .createAmResource(remoteConfPath, LocalResourceType.FILE); + localResources.put(localFilePath.getName(), localResource); + hasSliderAMLog4j = true; + } + } + return hasSliderAMLog4j; + } + + private void persistApp(Application application) + throws IOException, SliderException { + Path appDir = sliderFileSystem.buildClusterDirPath(application.getName()); + sliderFileSystem.verifyDirectoryNonexistent(appDir); + FsPermission permission = + sliderFileSystem.getInstanceDirectoryPermissions(); + sliderFileSystem.createWithPermissions(appDir, permission); + Path appJson = new Path(appDir, application.getName() + ".json"); + jsonSerDeser + .save(sliderFileSystem.getFileSystem(), appJson, application, true); + log.info( + "Persisted application " + application.getName() + " at " + appJson); + } + + private void addKeytabResourceIfSecure(SliderFileSystem fileSystem, + Map localResource, Configuration conf, + String appName) throws IOException, BadConfigException { + if (!UserGroupInformation.isSecurityEnabled()) { + return; + } + String keytabPreInstalledOnHost = + conf.get(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH); + if (StringUtils.isEmpty(keytabPreInstalledOnHost)) { + String amKeytabName = + conf.get(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME); + String keytabDir = conf.get(SliderXmlConfKeys.KEY_HDFS_KEYTAB_DIR); + Path keytabPath = + fileSystem.buildKeytabPath(keytabDir, amKeytabName, appName); + if (fileSystem.getFileSystem().exists(keytabPath)) { + LocalResource keytabRes = + fileSystem.createAmResource(keytabPath, LocalResourceType.FILE); + localResource + .put(SliderKeys.KEYTAB_DIR + "/" + amKeytabName, keytabRes); + log.info("Adding AM keytab on hdfs: " + keytabPath); + } else { + log.warn("No keytab file was found at {}.", keytabPath); + if (conf.getBoolean(KEY_AM_LOGIN_KEYTAB_REQUIRED, false)) { + throw new BadConfigException("No keytab file was found at %s.", + keytabPath); + } else { + log.warn("The AM will be " + + "started without a kerberos authenticated identity. " + + "The application is therefore not guaranteed to remain " + + "operational beyond 24 hours."); + } + } } - return startCluster(clustername, createArgs, createArgs.lifetime); } @Override @@ -781,7 +970,7 @@ public int actionUpgrade(String clustername, ActionUpgradeArgs upgradeArgs) } // Now initiate the upgrade spec flow - buildInstanceDefinition(clustername, upgradeArgs, true, true, true); +// buildInstanceDefinition(clustername, upgradeArgs, true, true, true); SliderClusterOperations clusterOperations = createClusterOperations(clustername); clusterOperations.amSuicide("AM restarted for application upgrade", 1, 1000); return EXIT_SUCCESS; @@ -797,7 +986,6 @@ public int actionUpgrade(String clustername, ActionUpgradeArgs upgradeArgs) private int actionUpgradeContainers(String clustername, ActionUpgradeArgs upgradeArgs) throws YarnException, IOException { - verifyBindingsDefined(); validateClusterName(clustername); int waittime = upgradeArgs.getWaittime(); // ignored for now String text = "Upgrade containers"; @@ -952,15 +1140,6 @@ protected static void checkForCredentials(Configuration conf, } } - private static char[] readOnePassword(String alias) throws IOException { - Console console = System.console(); - if (console == null) { - throw new IOException("Unable to input password for " + alias + - " because System.console() is null"); - } - return readPassword(alias, console); - } - private static char[] readPassword(String alias, Console console) throws IOException { char[] cred = null; @@ -988,12 +1167,10 @@ protected static void checkForCredentials(Configuration conf, @Override public int actionBuild(String clustername, - AbstractClusterBuildingActionArgs buildInfo) throws - YarnException, - IOException { - - buildInstanceDefinition(clustername, buildInfo, false, false); - return EXIT_SUCCESS; + AbstractClusterBuildingActionArgs buildInfo) + throws YarnException, IOException { + //TODO + return EXIT_SUCCESS; } @Override @@ -1527,12 +1704,12 @@ public int actionUpdate(String clustername, if (buildInfo.lifetime > 0) { updateLifetime(clustername, buildInfo.lifetime); } else { - buildInstanceDefinition(clustername, buildInfo, true, true); + //TODO upgrade } return EXIT_SUCCESS; } - public void updateLifetime(String appName, long lifetime) + public String updateLifetime(String appName, long lifetime) throws YarnException, IOException { EnumSet appStates = EnumSet.range( YarnApplicationState.NEW, YarnApplicationState.RUNNING); @@ -1553,396 +1730,7 @@ public void updateLifetime(String appName, long lifetime) log.info("Successfully updated lifetime for an application: appName = " + appName + ", appId = " + appId + ". New expiry time in ISO8601 format is " + newTimeout); - } - - /** - * Build up the AggregateConfiguration for an application instance then - * persists it - * @param clustername name of the cluster - * @param buildInfo the arguments needed to build the cluster - * @param overwrite true if existing cluster directory can be overwritten - * @param liveClusterAllowed true if live cluster can be modified - * @throws YarnException - * @throws IOException - */ - - public void buildInstanceDefinition(String clustername, - AbstractClusterBuildingActionArgs buildInfo, boolean overwrite, - boolean liveClusterAllowed) throws YarnException, IOException { - buildInstanceDefinition(clustername, buildInfo, overwrite, - liveClusterAllowed, false); - } - - public void buildInstanceDefinition(String clustername, - AbstractClusterBuildingActionArgs buildInfo, boolean overwrite, - boolean liveClusterAllowed, boolean isUpgradeFlow) throws YarnException, - IOException { - // verify that a live cluster isn't there - validateClusterName(clustername); - verifyBindingsDefined(); - if (!liveClusterAllowed) { - verifyNoLiveClusters(clustername, "Create"); - } - - Configuration conf = getConfig(); - String registryQuorum = lookupZKQuorum(); - - Path appconfdir = buildInfo.getConfdir(); - // Provider - String providerName = buildInfo.getProvider(); - requireArgumentSet(Arguments.ARG_PROVIDER, providerName); - log.debug("Provider is {}", providerName); - SliderAMClientProvider sliderAM = new SliderAMClientProvider(conf); - AbstractClientProvider provider = - createClientProvider(providerName); - InstanceBuilder builder = - new InstanceBuilder(sliderFileSystem, - getConfig(), - clustername); - - AggregateConf instanceDefinition = new AggregateConf(); - ConfTreeOperations appConf = instanceDefinition.getAppConfOperations(); - ConfTreeOperations resources = instanceDefinition.getResourceOperations(); - ConfTreeOperations internal = instanceDefinition.getInternalOperations(); - //initial definition is set by the providers - sliderAM.prepareInstanceConfiguration(instanceDefinition); - provider.prepareInstanceConfiguration(instanceDefinition); - - //load in any specified on the command line - if (buildInfo.resources != null) { - try { - resources.mergeFile(buildInfo.resources, - new ResourcesInputPropertiesValidator()); - - } catch (IOException e) { - throw new BadConfigException(e, - "incorrect argument to %s: \"%s\" : %s ", - Arguments.ARG_RESOURCES, - buildInfo.resources, - e.toString()); - } - } - if (buildInfo.template != null) { - try { - appConf.mergeFile(buildInfo.template, - new TemplateInputPropertiesValidator()); - } catch (IOException e) { - throw new BadConfigException(e, - "incorrect argument to %s: \"%s\" : %s ", - Arguments.ARG_TEMPLATE, - buildInfo.template, - e.toString()); - } - } - - if (isUpgradeFlow) { - ActionUpgradeArgs upgradeInfo = (ActionUpgradeArgs) buildInfo; - if (!upgradeInfo.force) { - validateClientAndClusterResource(clustername, resources); - } - } - - //get the command line options - ConfTree cmdLineAppOptions = buildInfo.buildAppOptionsConfTree(); - ConfTree cmdLineResourceOptions = buildInfo.buildResourceOptionsConfTree(); - - appConf.merge(cmdLineAppOptions); - - AppDefinitionPersister appDefinitionPersister = new AppDefinitionPersister(sliderFileSystem); - appDefinitionPersister.processSuppliedDefinitions(clustername, buildInfo, appConf); - - // put the role counts into the resources file - Map argsRoleMap = buildInfo.getComponentMap(); - for (Map.Entry roleEntry : argsRoleMap.entrySet()) { - String count = roleEntry.getValue(); - String key = roleEntry.getKey(); - log.info("{} => {}", key, count); - resources.getOrAddComponent(key).put(COMPONENT_INSTANCES, count); - } - - //all CLI role options - Map> appOptionMap = - buildInfo.getCompOptionMap(); - appConf.mergeComponents(appOptionMap); - - //internal picks up core. values only - internal.propagateGlobalKeys(appConf, "slider."); - internal.propagateGlobalKeys(appConf, "internal."); - - //copy over role. and yarn. values ONLY to the resources - if (PROPAGATE_RESOURCE_OPTION) { - resources.propagateGlobalKeys(appConf, "component."); - resources.propagateGlobalKeys(appConf, "role."); - resources.propagateGlobalKeys(appConf, "yarn."); - resources.mergeComponentsPrefix(appOptionMap, "component.", true); - resources.mergeComponentsPrefix(appOptionMap, "yarn.", true); - resources.mergeComponentsPrefix(appOptionMap, "role.", true); - } - - // resource component args - appConf.merge(cmdLineResourceOptions); - resources.merge(cmdLineResourceOptions); - resources.mergeComponents(buildInfo.getResourceCompOptionMap()); - - builder.init(providerName, instanceDefinition); - builder.resolve(); - builder.propagateFilename(); - builder.propagatePrincipals(); - builder.setImageDetailsIfAvailable(buildInfo.getImage(), - buildInfo.getAppHomeDir()); - builder.setQueue(buildInfo.queue); - - String quorum = buildInfo.getZKhosts(); - if (isUnset(quorum)) { - quorum = registryQuorum; - } - if (isUnset(quorum)) { - throw new BadConfigException(E_NO_ZOOKEEPER_QUORUM); - } - ZKPathBuilder zkPaths = new ZKPathBuilder(getAppName(), - getUsername(), - clustername, - registryQuorum, - quorum); - String zookeeperRoot = buildInfo.getAppZKPath(); - - if (isSet(zookeeperRoot)) { - zkPaths.setAppPath(zookeeperRoot); - } else { - String createDefaultZkNode = appConf.getGlobalOptions() - .getOption(AgentKeys.CREATE_DEF_ZK_NODE, "false"); - if (createDefaultZkNode.equals("true")) { - String defaultZKPath = createZookeeperNode(clustername, false); - log.debug("ZK node created for application instance: {}", defaultZKPath); - if (defaultZKPath != null) { - zkPaths.setAppPath(defaultZKPath); - } - } else { - // create AppPath if default is being used - String defaultZKPath = createZookeeperNode(clustername, true); - log.debug("ZK node assigned to application instance: {}", defaultZKPath); - zkPaths.setAppPath(defaultZKPath); - } - } - - builder.addZKBinding(zkPaths); - - //then propagate any package URI - if (buildInfo.packageURI != null) { - appConf.set(AgentKeys.PACKAGE_PATH, buildInfo.packageURI); - } - - propagatePythonExecutable(conf, instanceDefinition); - - // make any substitutions needed at this stage - replaceTokens(appConf.getConfTree(), getUsername(), clustername); - - // TODO: Refactor the validation code and persistence code - try { - persistInstanceDefinition(overwrite, appconfdir, builder); - appDefinitionPersister.persistPackages(); - - } catch (LockAcquireFailedException e) { - log.warn("Failed to get a Lock on {} : {}", builder, e, e); - throw new BadClusterStateException("Failed to save " + clustername - + ": " + e); - } - - // providers to validate what there is - // TODO: Validation should be done before persistence - AggregateConf instanceDescription = builder.getInstanceDescription(); - validateInstanceDefinition(sliderAM, instanceDescription, sliderFileSystem); - validateInstanceDefinition(provider, instanceDescription, sliderFileSystem); - } - - private void validateClientAndClusterResource(String clustername, - ConfTreeOperations clientResources) throws BadClusterStateException, - SliderException, IOException { - log.info("Validating upgrade resource definition with current cluster " - + "state (components and instance count)"); - Map clientComponentInstances = new HashMap<>(); - for (String componentName : clientResources.getComponentNames()) { - if (!SliderKeys.COMPONENT_AM.equals(componentName)) { - clientComponentInstances.put(componentName, clientResources - .getComponentOptInt(componentName, - COMPONENT_INSTANCES, -1)); - } - } - - AggregateConf clusterConf = null; - try { - clusterConf = loadPersistedClusterDescription(clustername); - } catch (LockAcquireFailedException e) { - log.warn("Failed to get a Lock on cluster resource : {}", e, e); - throw new BadClusterStateException( - "Failed to load client resource definition " + clustername + ": " + e, e); - } - Map clusterComponentInstances = new HashMap<>(); - for (Map.Entry> component : clusterConf - .getResources().components.entrySet()) { - if (!SliderKeys.COMPONENT_AM.equals(component.getKey())) { - clusterComponentInstances.put( - component.getKey(), - Integer.decode(component.getValue().get( - COMPONENT_INSTANCES))); - } - } - - // client and cluster should be an exact match - Iterator> clientComponentInstanceIt = clientComponentInstances - .entrySet().iterator(); - while (clientComponentInstanceIt.hasNext()) { - Map.Entry clientComponentInstanceEntry = clientComponentInstanceIt.next(); - if (clusterComponentInstances.containsKey(clientComponentInstanceEntry.getKey())) { - // compare instance count now and remove from both maps if they match - if (clusterComponentInstances - .get(clientComponentInstanceEntry.getKey()).intValue() == clientComponentInstanceEntry - .getValue().intValue()) { - clusterComponentInstances.remove(clientComponentInstanceEntry - .getKey()); - clientComponentInstanceIt.remove(); - } - } - } - - if (!clientComponentInstances.isEmpty() - || !clusterComponentInstances.isEmpty()) { - log.error("Mismatch found in upgrade resource definition and cluster " - + "resource state"); - if (!clientComponentInstances.isEmpty()) { - log.info("The upgrade resource definitions that do not match are:"); - for (Map.Entry clientComponentInstanceEntry : clientComponentInstances - .entrySet()) { - log.info(" Component Name: {}, Instance count: {}", - clientComponentInstanceEntry.getKey(), - clientComponentInstanceEntry.getValue()); - } - } - if (!clusterComponentInstances.isEmpty()) { - log.info("The cluster resources that do not match are:"); - for (Map.Entry clusterComponentInstanceEntry : clusterComponentInstances - .entrySet()) { - log.info(" Component Name: {}, Instance count: {}", - clusterComponentInstanceEntry.getKey(), - clusterComponentInstanceEntry.getValue()); - } - } - throw new BadConfigException("Resource definition provided for " - + "upgrade does not match with that of the currently running " - + "cluster.\nIf you are aware of what you are doing, rerun the " - + "command with " + Arguments.ARG_FORCE + " option."); - } - } - - protected void persistInstanceDefinition(boolean overwrite, - Path appconfdir, - InstanceBuilder builder) - throws IOException, SliderException, LockAcquireFailedException { - builder.persist(appconfdir, overwrite); - } - - @VisibleForTesting - public static void replaceTokens(ConfTree conf, - String userName, String clusterName) throws IOException { - Map newglobal = new HashMap<>(); - for (Entry entry : conf.global.entrySet()) { - newglobal.put(entry.getKey(), replaceTokens(entry.getValue(), - userName, clusterName)); - } - conf.global.putAll(newglobal); - - for (String component : conf.components.keySet()) { - Map newComponent = new HashMap<>(); - for (Entry entry : conf.components.get(component).entrySet()) { - newComponent.put(entry.getKey(), replaceTokens(entry.getValue(), - userName, clusterName)); - } - conf.components.get(component).putAll(newComponent); - } - - Map> newcred = new HashMap<>(); - for (Entry> entry : conf.credentials.entrySet()) { - List resultList = new ArrayList<>(); - for (String v : entry.getValue()) { - resultList.add(replaceTokens(v, userName, clusterName)); - } - newcred.put(replaceTokens(entry.getKey(), userName, clusterName), - resultList); - } - conf.credentials.clear(); - conf.credentials.putAll(newcred); - } - - private static String replaceTokens(String s, String userName, - String clusterName) throws IOException { - return s.replaceAll(Pattern.quote("${USER}"), userName) - .replaceAll(Pattern.quote("${USER_NAME}"), userName); - } - - public FsPermission getClusterDirectoryPermissions(Configuration conf) { - String clusterDirPermsOct = - conf.get(CLUSTER_DIRECTORY_PERMISSIONS, DEFAULT_CLUSTER_DIRECTORY_PERMISSIONS); - return new FsPermission(clusterDirPermsOct); - } - - /** - * Verify that the Resource Manager is configured (on a non-HA cluster). - * with a useful error message - * @throws BadCommandArgumentsException the exception raised on an invalid config - */ - public void verifyBindingsDefined() throws BadCommandArgumentsException { - InetSocketAddress rmAddr = getRmAddress(getConfig()); - if (!getConfig().getBoolean(YarnConfiguration.RM_HA_ENABLED, false) - && !isAddressDefined(rmAddr)) { - throw new BadCommandArgumentsException( - E_NO_RESOURCE_MANAGER - + " in the argument " - + Arguments.ARG_MANAGER - + " or the configuration property " - + YarnConfiguration.RM_ADDRESS - + " value :" + rmAddr); - } - } - - /** - * Load and start a cluster specification. - * This assumes that all validation of args and cluster state - * have already taken place - * - * @param clustername name of the cluster. - * @param launchArgs launch arguments - * @param lifetime - * @return the exit code - * @throws YarnException - * @throws IOException - */ - protected int startCluster(String clustername, LaunchArgsAccessor launchArgs, - long lifetime) throws YarnException, IOException { - Path clusterDirectory = sliderFileSystem.buildClusterDirPath(clustername); - AggregateConf instanceDefinition = loadInstanceDefinitionUnresolved( - clustername, - clusterDirectory); - - LaunchedApplication launchedApplication = - launchApplication(clustername, clusterDirectory, instanceDefinition, - serviceArgs.isDebug(), lifetime); - - if (launchArgs.getOutputFile() != null) { - // output file has been requested. Get the app report and serialize it - ApplicationReport report = - launchedApplication.getApplicationReport(); - SerializedApplicationReport sar = new SerializedApplicationReport(report); - sar.submitTime = System.currentTimeMillis(); - ApplicationReportSerDeser serDeser = new ApplicationReportSerDeser(); - serDeser.save(sar, launchArgs.getOutputFile()); - } - int waittime = launchArgs.getWaittime(); - if (waittime > 0) { - return waitForAppRunning(launchedApplication, waittime, waittime); - } else { - // no waiting - return EXIT_SUCCESS; - } + return newTimeout; } /** @@ -1968,415 +1756,6 @@ public AggregateConf loadInstanceDefinitionUnresolved(String name, } } - /** - * Load the instance definition. - * @param name cluster name - * @param resolved flag to indicate the cluster should be resolved - * @return the loaded configuration - * @throws IOException IO problems - * @throws SliderException slider explicit issues - * @throws UnknownApplicationInstanceException if the file is not found - */ - public AggregateConf loadInstanceDefinition(String name, - boolean resolved) throws - IOException, - SliderException { - - Path clusterDirectory = sliderFileSystem.buildClusterDirPath(name); - AggregateConf instanceDefinition = loadInstanceDefinitionUnresolved( - name, - clusterDirectory); - if (resolved) { - instanceDefinition.resolve(); - } - return instanceDefinition; - - } - - protected AppMasterLauncher setupAppMasterLauncher(String clustername, - Path clusterDirectory, AggregateConf instanceDefinition, boolean debugAM, - long lifetime) - throws YarnException, IOException{ - deployedClusterName = clustername; - validateClusterName(clustername); - verifyNoLiveClusters(clustername, "Launch"); - Configuration config = getConfig(); - lookupZKQuorum(); - boolean clusterSecure = isHadoopClusterSecure(config); - //create the Slider AM provider -this helps set up the AM - SliderAMClientProvider sliderAM = new SliderAMClientProvider(config); - - instanceDefinition.resolve(); - launchedInstanceDefinition = instanceDefinition; - - ConfTreeOperations internalOperations = instanceDefinition.getInternalOperations(); - MapOperations internalOptions = internalOperations.getGlobalOptions(); - ConfTreeOperations resourceOperations = instanceDefinition.getResourceOperations(); - ConfTreeOperations appOperations = instanceDefinition.getAppConfOperations(); - Path generatedConfDirPath = - createPathThatMustExist(internalOptions.getMandatoryOption( - INTERNAL_GENERATED_CONF_PATH)); - Path snapshotConfPath = - createPathThatMustExist(internalOptions.getMandatoryOption( - INTERNAL_SNAPSHOT_CONF_PATH)); - - - // cluster Provider - AbstractClientProvider provider = createClientProvider( - internalOptions.getMandatoryOption(INTERNAL_PROVIDER_NAME)); - if (log.isDebugEnabled()) { - log.debug(instanceDefinition.toString()); - } - MapOperations sliderAMResourceComponent = - resourceOperations.getOrAddComponent(SliderKeys.COMPONENT_AM); - MapOperations resourceGlobalOptions = resourceOperations.getGlobalOptions(); - - // add the tags if available - Set applicationTags = provider.getApplicationTags(sliderFileSystem, - appOperations, clustername); - - Credentials credentials = null; - if (clusterSecure) { - // pick up oozie credentials - credentials = CredentialUtils.loadTokensFromEnvironment(System.getenv(), - config); - if (credentials == null) { - // nothing from oozie, so build up directly - credentials = new Credentials( - UserGroupInformation.getCurrentUser().getCredentials()); - CredentialUtils.addRMRenewableFSDelegationTokens(config, - sliderFileSystem.getFileSystem(), - credentials); - CredentialUtils.addRMDelegationToken(yarnClient, credentials); - - } else { - log.info("Using externally supplied credentials to launch AM"); - } - } - - AppMasterLauncher amLauncher = new AppMasterLauncher(clustername, - SliderKeys.APP_TYPE, - config, - sliderFileSystem, - yarnClient, - clusterSecure, - sliderAMResourceComponent, - resourceGlobalOptions, - applicationTags, - credentials); - - ApplicationId appId = amLauncher.getApplicationId(); - // set the application name; - amLauncher.setKeepContainersOverRestarts(true); - // set lifetime in submission context; - Map appTimeout = new HashMap<>(); - if (lifetime > 0) { - appTimeout.put(ApplicationTimeoutType.LIFETIME, lifetime); - } - amLauncher.submissionContext.setApplicationTimeouts(appTimeout); - int maxAppAttempts = config.getInt(KEY_AM_RESTART_LIMIT, 0); - amLauncher.setMaxAppAttempts(maxAppAttempts); - - sliderFileSystem.purgeAppInstanceTempFiles(clustername); - Path tempPath = sliderFileSystem.createAppInstanceTempPath( - clustername, - appId.toString() + "/am"); - String libdir = "lib"; - Path libPath = new Path(tempPath, libdir); - sliderFileSystem.getFileSystem().mkdirs(libPath); - log.debug("FS={}, tempPath={}, libdir={}", sliderFileSystem, tempPath, libPath); - - // set local resources for the application master - // local files or archives as needed - // In this scenario, the jar file for the application master is part of the local resources - Map localResources = amLauncher.getLocalResources(); - - // look for the configuration directory named on the command line - boolean hasServerLog4jProperties = false; - Path remoteConfPath = null; - String relativeConfDir = null; - String confdirProp = System.getProperty(SliderKeys.PROPERTY_CONF_DIR); - if (isUnset(confdirProp)) { - log.debug("No local configuration directory provided as system property"); - } else { - File confDir = new File(confdirProp); - if (!confDir.exists()) { - throw new BadConfigException(E_CONFIGURATION_DIRECTORY_NOT_FOUND, - confDir); - } - Path localConfDirPath = createLocalPath(confDir); - remoteConfPath = new Path(clusterDirectory, SliderKeys.SUBMITTED_CONF_DIR); - log.debug("Slider configuration directory is {}; remote to be {}", - localConfDirPath, remoteConfPath); - copyDirectory(config, localConfDirPath, remoteConfPath, null); - - File log4jserver = - new File(confDir, SliderKeys.LOG4J_SERVER_PROP_FILENAME); - hasServerLog4jProperties = log4jserver.isFile(); - } - if (!hasServerLog4jProperties) { - // check for log4j properties in hadoop conf dir - String hadoopConfDir = System.getenv(ApplicationConstants.Environment - .HADOOP_CONF_DIR.name()); - if (hadoopConfDir != null) { - File localFile = new File(hadoopConfDir, SliderKeys - .LOG4J_SERVER_PROP_FILENAME); - if (localFile.exists()) { - Path localFilePath = createLocalPath(localFile); - remoteConfPath = new Path(clusterDirectory, - SliderKeys.SUBMITTED_CONF_DIR); - Path remoteFilePath = new Path(remoteConfPath, SliderKeys - .LOG4J_SERVER_PROP_FILENAME); - copy(config, localFilePath, remoteFilePath); - hasServerLog4jProperties = true; - } - } - } - // the assumption here is that minimr cluster => this is a test run - // and the classpath can look after itself - - boolean usingMiniMRCluster = getUsingMiniMRCluster(); - if (!usingMiniMRCluster) { - - log.debug("Destination is not a MiniYARNCluster -copying full classpath"); - - // insert conf dir first - if (remoteConfPath != null) { - relativeConfDir = SliderKeys.SUBMITTED_CONF_DIR; - Map submittedConfDir = - sliderFileSystem.submitDirectory(remoteConfPath, - relativeConfDir); - mergeMaps(localResources, submittedConfDir); - } - } - // build up the configuration - // IMPORTANT: it is only after this call that site configurations - // will be valid. - - propagatePrincipals(config, instanceDefinition); - // validate security data - -/* - // turned off until tested - SecurityConfiguration securityConfiguration = - new SecurityConfiguration(config, - instanceDefinition, clustername); - -*/ - Configuration clientConfExtras = new Configuration(false); - // then build up the generated path. - FsPermission clusterPerms = getClusterDirectoryPermissions(config); - copyDirectory(config, snapshotConfPath, generatedConfDirPath, - clusterPerms); - - - // standard AM resources - sliderAM.prepareAMAndConfigForLaunch(sliderFileSystem, - config, - amLauncher, - instanceDefinition, - snapshotConfPath, - generatedConfDirPath, - clientConfExtras, - libdir, - tempPath, - usingMiniMRCluster); - //add provider-specific resources - provider.prepareAMAndConfigForLaunch(sliderFileSystem, - config, - amLauncher, - instanceDefinition, - snapshotConfPath, - generatedConfDirPath, - clientConfExtras, - libdir, - tempPath, - usingMiniMRCluster); - - // now that the site config is fully generated, the provider gets - // to do a quick review of them. - log.debug("Preflight validation of cluster configuration"); - - - sliderAM.preflightValidateClusterConfiguration(sliderFileSystem, - clustername, - config, - instanceDefinition, - clusterDirectory, - generatedConfDirPath, - clusterSecure - ); - - provider.preflightValidateClusterConfiguration(sliderFileSystem, - clustername, - config, - instanceDefinition, - clusterDirectory, - generatedConfDirPath, - clusterSecure - ); - - - if (!(provider instanceof DockerClientProvider)) { - Path imagePath = - extractImagePath(sliderFileSystem, internalOptions); - if (sliderFileSystem.maybeAddImagePath(localResources, imagePath)) { - log.debug("Registered image path {}", imagePath); - } - } - - // build the environment - amLauncher.putEnv( - buildEnvMap(sliderAMResourceComponent)); - ClasspathConstructor classpath = buildClasspath(relativeConfDir, - libdir, - getConfig(), - sliderFileSystem, - usingMiniMRCluster); - amLauncher.setClasspath(classpath); - //add english env - amLauncher.setEnv("LANG", "en_US.UTF-8"); - amLauncher.setEnv("LC_ALL", "en_US.UTF-8"); - amLauncher.setEnv("LANGUAGE", "en_US.UTF-8"); - amLauncher.maybeSetEnv(HADOOP_JAAS_DEBUG, - System.getenv(HADOOP_JAAS_DEBUG)); - amLauncher.putEnv(getAmLaunchEnv(config)); - - for (Map.Entry envs : getSystemEnv().entrySet()) { - log.debug("System env {}={}", envs.getKey(), envs.getValue()); - } - if (log.isDebugEnabled()) { - log.debug("AM classpath={}", classpath); - log.debug("Environment Map:\n{}", - stringifyMap(amLauncher.getEnv())); - log.debug("Files in lib path\n{}", sliderFileSystem.listFSDir(libPath)); - } - - // rm address - - InetSocketAddress rmSchedulerAddress; - try { - rmSchedulerAddress = getRmSchedulerAddress(config); - } catch (IllegalArgumentException e) { - throw new BadConfigException("%s Address invalid: %s", - YarnConfiguration.RM_SCHEDULER_ADDRESS, - config.get(YarnConfiguration.RM_SCHEDULER_ADDRESS)); - } - String rmAddr = NetUtils.getHostPortString(rmSchedulerAddress); - - JavaCommandLineBuilder commandLine = new JavaCommandLineBuilder(); - // insert any JVM options); - sliderAM.addJVMOptions(instanceDefinition, commandLine); - // enable asserts - commandLine.enableJavaAssertions(); - - // if the conf dir has a slideram-log4j.properties, switch to that - if (hasServerLog4jProperties) { - commandLine.sysprop(SYSPROP_LOG4J_CONFIGURATION, LOG4J_SERVER_PROP_FILENAME); - commandLine.sysprop(SYSPROP_LOG_DIR, ApplicationConstants.LOG_DIR_EXPANSION_VAR); - } - - // add the AM sevice entry point - commandLine.add(SliderAppMaster.SERVICE_CLASSNAME); - - // create action and the cluster name - commandLine.add(ACTION_CREATE, clustername); - - // debug - if (debugAM) { - commandLine.add(Arguments.ARG_DEBUG); - } - - // set the cluster directory path - commandLine.add(Arguments.ARG_CLUSTER_URI, clusterDirectory.toUri()); - - if (!isUnset(rmAddr)) { - commandLine.add(Arguments.ARG_RM_ADDR, rmAddr); - } - - if (serviceArgs.getFilesystemBinding() != null) { - commandLine.add(Arguments.ARG_FILESYSTEM, serviceArgs.getFilesystemBinding()); - } - - // pass the registry binding - commandLine.addConfOptionToCLI(config, RegistryConstants.KEY_REGISTRY_ZK_ROOT, - RegistryConstants.DEFAULT_ZK_REGISTRY_ROOT); - commandLine.addMandatoryConfOption(config, RegistryConstants.KEY_REGISTRY_ZK_QUORUM); - - if (clusterSecure) { - // if the cluster is secure, make sure that - // the relevant security settings go over - commandLine.addConfOption(config, DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY); - } - - // copy over any/all YARN RM client values, in case the server-side XML conf file - // has the 0.0.0.0 address - commandLine.addConfOptions(config, - YarnConfiguration.RM_ADDRESS, - YarnConfiguration.RM_CLUSTER_ID, - YarnConfiguration.RM_HOSTNAME, - YarnConfiguration.RM_PRINCIPAL); - - // write out the path output - commandLine.addOutAndErrFiles(STDOUT_AM, STDERR_AM); - - String cmdStr = commandLine.build(); - log.debug("Completed setting up app master command {}", cmdStr); - - amLauncher.addCommandLine(commandLine); - - // the Slider AM gets to configure the AM requirements, not the custom provider - sliderAM.prepareAMResourceRequirements(sliderAMResourceComponent, - amLauncher.getResource()); - - - // Set the priority for the application master - amLauncher.setPriority(config.getInt(KEY_YARN_QUEUE_PRIORITY, - DEFAULT_YARN_QUEUE_PRIORITY)); - - // Set the queue to which this application is to be submitted in the RM - // Queue for App master - String amQueue = config.get(KEY_YARN_QUEUE, DEFAULT_YARN_QUEUE); - String suppliedQueue = internalOperations.getGlobalOptions().get(INTERNAL_QUEUE); - if(!isUnset(suppliedQueue)) { - amQueue = suppliedQueue; - log.info("Using queue {} for the application instance.", amQueue); - } - - if (isSet(amQueue)) { - amLauncher.setQueue(amQueue); - } - return amLauncher; - } - - /** - * - * @param clustername name of the cluster - * @param clusterDirectory cluster dir - * @param instanceDefinition the instance definition - * @param debugAM enable debug AM options - * @param lifetime - * @return the launched application - * @throws YarnException - * @throws IOException - */ - public LaunchedApplication launchApplication(String clustername, Path clusterDirectory, - AggregateConf instanceDefinition, boolean debugAM, long lifetime) - throws YarnException, IOException { - - AppMasterLauncher amLauncher = setupAppMasterLauncher(clustername, - clusterDirectory, - instanceDefinition, - debugAM, lifetime); - - applicationId = amLauncher.getApplicationId(); - log.info("Submitting application {}", applicationId); - - // submit the application - LaunchedApplication launchedApplication = amLauncher.submitApplication(); - return launchedApplication; - } - protected Map getAmLaunchEnv(Configuration config) { String sliderAmLaunchEnv = config.get(KEY_AM_LAUNCH_ENV); log.debug("{} = {}", KEY_AM_LAUNCH_ENV, sliderAmLaunchEnv); @@ -2431,95 +1810,6 @@ public LaunchedApplication launchApplication(String clustername, Path clusterDir return placeholderKeyValueMap; } - private void propagatePythonExecutable(Configuration config, - AggregateConf instanceDefinition) { - String pythonExec = config.get( - PYTHON_EXECUTABLE_PATH); - if (pythonExec != null) { - instanceDefinition.getAppConfOperations().getGlobalOptions().putIfUnset( - PYTHON_EXECUTABLE_PATH, - pythonExec); - } - } - - - /** - * Wait for the launched app to be accepted in the time - * and, optionally running. - *

- * If the application - * - * @param launchedApplication application - * @param acceptWaitMillis time in millis to wait for accept - * @param runWaitMillis time in millis to wait for the app to be running. - * May be null, in which case no wait takes place - * @return exit code: success - * @throws YarnException - * @throws IOException - */ - public int waitForAppRunning(LaunchedApplication launchedApplication, - int acceptWaitMillis, int runWaitMillis) throws YarnException, IOException { - assert launchedApplication != null; - int exitCode; - // wait for the submit state to be reached - ApplicationReport report = launchedApplication.monitorAppToState( - YarnApplicationState.ACCEPTED, - new Duration(acceptWaitMillis)); - - // may have failed, so check that - if (hasAppFinished(report)) { - exitCode = buildExitCode(report); - } else { - // exit unless there is a wait - - - if (runWaitMillis != 0) { - // waiting for state to change - Duration duration = new Duration(runWaitMillis * 1000); - duration.start(); - report = launchedApplication.monitorAppToState( - YarnApplicationState.RUNNING, duration); - if (report != null && - report.getYarnApplicationState() == YarnApplicationState.RUNNING) { - exitCode = EXIT_SUCCESS; - } else { - exitCode = buildExitCode(report); - } - } else { - exitCode = EXIT_SUCCESS; - } - } - return exitCode; - } - - - /** - * Propagate any critical principals from the current site config down to the HBase one. - * @param config config to read from - * @param clusterSpec cluster spec - */ - private void propagatePrincipals(Configuration config, - AggregateConf clusterSpec) { - String dfsPrincipal = config.get(DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY); - if (dfsPrincipal != null) { - String siteDfsPrincipal = SITE_XML_PREFIX + DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY; - clusterSpec.getAppConfOperations().getGlobalOptions().putIfUnset( - siteDfsPrincipal, - dfsPrincipal); - } - } - - /** - * Create a path that must exist in the cluster fs - * @param uri uri to create - * @return the path - * @throws FileNotFoundException if the path does not exist - */ - public Path createPathThatMustExist(String uri) throws - SliderException, IOException { - return sliderFileSystem.createPathThatMustExist(uri); - } - /** * verify that a live cluster isn't there * @param clustername cluster name @@ -2554,11 +1844,6 @@ public String getDeployedClusterName() { return deployedClusterName; } - @VisibleForTesting - public void setDeployedClusterName(String deployedClusterName) { - this.deployedClusterName = deployedClusterName; - } - /** * ask if the client is using a mini MR cluster * @return true if they are @@ -2568,109 +1853,6 @@ private boolean getUsingMiniMRCluster() { false); } - /** - * Get the application name used in the zookeeper root paths - * @return an application-specific path in ZK - */ - private String getAppName() { - return "slider"; - } - - /** - * Wait for the app to start running (or go past that state) - * @param duration time to wait - * @return the app report; null if the duration turned out - * @throws YarnException YARN or app issues - * @throws IOException IO problems - */ - @VisibleForTesting - public ApplicationReport monitorAppToRunning(Duration duration) - throws YarnException, IOException { - return monitorAppToState(YarnApplicationState.RUNNING, duration); - } - - /** - * Build an exit code for an application from its report. - * If the report parameter is null, its interpreted as a timeout - * @param report report application report - * @return the exit code - * @throws IOException - * @throws YarnException - */ - private int buildExitCode(ApplicationReport report) throws - IOException, - YarnException { - if (null == report) { - return EXIT_TIMED_OUT; - } - - YarnApplicationState state = report.getYarnApplicationState(); - FinalApplicationStatus dsStatus = report.getFinalApplicationStatus(); - switch (state) { - case FINISHED: - if (FinalApplicationStatus.SUCCEEDED == dsStatus) { - log.info("Application has completed successfully"); - return EXIT_SUCCESS; - } else { - log.info("Application finished unsuccessfully." + - "YarnState = {}, DSFinalStatus = {} Breaking monitoring loop", - state, dsStatus); - return EXIT_YARN_SERVICE_FINISHED_WITH_ERROR; - } - - case KILLED: - log.info("Application did not finish. YarnState={}, DSFinalStatus={}", - state, dsStatus); - return EXIT_YARN_SERVICE_KILLED; - - case FAILED: - log.info("Application Failed. YarnState={}, DSFinalStatus={}", state, - dsStatus); - return EXIT_YARN_SERVICE_FAILED; - - default: - //not in any of these states - return EXIT_SUCCESS; - } - } - - /** - * Monitor the submitted application for reaching the requested state. - * Will also report if the app reaches a later state (failed, killed, etc) - * Kill application if duration!= null & time expires. - * Prerequisite: the applicatin was launched. - * @param desiredState desired state. - * @param duration how long to wait -must be more than 0 - * @return the application report -null on a timeout - * @throws YarnException - * @throws IOException - */ - @VisibleForTesting - public ApplicationReport monitorAppToState( - YarnApplicationState desiredState, - Duration duration) - throws YarnException, IOException { - LaunchedApplication launchedApplication = - new LaunchedApplication(applicationId, yarnClient); - return launchedApplication.monitorAppToState(desiredState, duration); - } - - @Override - public ApplicationReport getApplicationReport() throws - IOException, - YarnException { - return getApplicationReport(applicationId); - } - - @Override - public boolean forceKillApplication(String reason) - throws YarnException, IOException { - if (applicationId != null) { - new LaunchedApplication(applicationId, yarnClient).forceKill(reason); - return true; - } - return false; - } /** * List Slider instances belonging to a specific user with a specific app @@ -2721,23 +1903,6 @@ public int actionList(String clustername, ActionListArgs args) } /** - * Retrieve a list of all live instances. If clustername is supplied then it - * returns this specific cluster, if and only if it exists and is live. - * - * @param clustername - * cluster name (if looking for a specific live cluster) - * @return the list of application names which satisfies the list criteria - * @throws IOException - * @throws YarnException - */ - public Set getApplicationList(String clustername) - throws IOException, YarnException { - ActionListArgs args = new ActionListArgs(); - args.live = true; - return getApplicationList(clustername, args); - } - - /** * Retrieve a list of application instances satisfying the query criteria. * * @param clustername @@ -2757,8 +1922,6 @@ public int actionList(String clustername, ActionListArgs args) // the above call throws an exception so the return is not really required return Collections.emptySet(); } - verifyBindingsDefined(); - boolean live = args.live; String state = args.state; boolean listContainers = args.containers; @@ -2868,29 +2031,6 @@ public int actionList(String clustername, ActionListArgs args) } } - /** - * Enumerate slider instances for the current user, and the - * most recent app report, where available. - * @param listOnlyInState boolean to indicate that the instances should - * only include those in a YARN state - * minAppState <= currentState <= maxAppState - * - * @param minAppState minimum application state to include in enumeration. - * @param maxAppState maximum application state to include - * @return a map of application instance name to description - * @throws IOException Any IO problem - * @throws YarnException YARN problems - */ - @Override - public Map enumSliderInstances( - boolean listOnlyInState, - YarnApplicationState minAppState, - YarnApplicationState maxAppState) - throws IOException, YarnException { - return yarnAppListClient.enumSliderInstances(listOnlyInState, - minAppState, - maxAppState); - } /** * Extract the state of a Yarn application --state argument @@ -2935,7 +2075,6 @@ public int actionFlex(String name, ActionFlexArgs args) throws YarnException, IO if (roleMap.size() == 0) { actionHelp(ACTION_FLEX); } - verifyBindingsDefined(); log.debug("actionFlex({})", name); Map roleInstances = new HashMap<>(); for (Map.Entry roleEntry : roleMap.entrySet()) { @@ -2943,7 +2082,7 @@ public int actionFlex(String name, ActionFlexArgs args) throws YarnException, IO String val = roleEntry.getValue(); roleInstances.put(key, val); } - return flex(name, roleInstances); + return 1; } @Override @@ -2954,7 +2093,6 @@ public int actionExists(String name, boolean checkLive) throws YarnException, IO } public int actionExists(String name, ActionExistsArgs args) throws YarnException, IOException { - verifyBindingsDefined(); validateClusterName(name); boolean checkLive = args.live; log.debug("actionExists({}, {}, {})", name, checkLive, args.state); @@ -3050,14 +2188,6 @@ public String actionEcho(String name, ActionEchoArgs args) throws } /** - * Get at the service registry operations - * @return registry client -valid after the service is inited. - */ - public YarnAppListClient getYarnAppListClient() { - return yarnAppListClient; - } - - /** * Find an instance of an application belonging to the current user. * @param appname application name * @return the app report or null if none is found @@ -3128,20 +2258,20 @@ public int actionStatus(String clustername, ActionStatusArgs statusArgs) return EXIT_SUCCESS; } - ClusterDescription status = verifyAndGetClusterDescription(clustername); + Application application = getApplication(clustername); String outfile = statusArgs.getOutput(); if (outfile == null) { - log.info(status.toJsonString()); + log.info(application.toString()); } else { - status.save(new File(outfile).getAbsoluteFile()); + jsonSerDeser.save(application, new File(statusArgs.getOutput())); } return EXIT_SUCCESS; } @Override - public String actionStatus(String clustername) + public Application actionStatus(String clustername) throws YarnException, IOException { - return verifyAndGetClusterDescription(clustername).toJsonString(); + return getApplication(clustername); } private void queryAndPrintLifetime(String appName) @@ -3170,13 +2300,6 @@ private void queryAndPrintLifetime(String appName) } } - private ClusterDescription verifyAndGetClusterDescription(String clustername) - throws YarnException, IOException { - verifyBindingsDefined(); - validateClusterName(clustername); - return getClusterDescription(clustername); - } - @Override public int actionVersion() { SliderVersionInfo.loadAndPrintVersionInfo(log); @@ -3184,269 +2307,98 @@ public int actionVersion() { } @Override - public int actionFreeze(String clustername, - ActionFreezeArgs freezeArgs) throws YarnException, IOException { - verifyBindingsDefined(); - validateClusterName(clustername); - int waittime = freezeArgs.getWaittime(); - String text = freezeArgs.message; - boolean forcekill = freezeArgs.force; - log.debug("actionFreeze({}, reason={}, wait={}, force={})", clustername, - text, - waittime, - forcekill); - - //is this actually a known cluster? - sliderFileSystem.locateInstanceDefinition(clustername); - ApplicationReport app = findInstance(clustername); + public void actionStop(String appName, ActionFreezeArgs freezeArgs) + throws YarnException, IOException { + validateClusterName(appName); + ApplicationReport app = findInstance(appName); if (app == null) { - // exit early - log.info("Cluster {} not running", clustername); - // not an error to stop a stopped cluster - return EXIT_SUCCESS; - } - log.debug("App to stop was found: {}:\n{}", clustername, - new OnDemandReportStringifier(app)); - if (app.getYarnApplicationState().ordinal() >= - YarnApplicationState.FINISHED.ordinal()) { - log.info("Cluster {} is in a terminated state {}", clustername, - app.getYarnApplicationState()); - return EXIT_SUCCESS; + throw new ApplicationNotFoundException( + "Application " + appName + " doesn't exist in RM."); } - // IPC request for a managed shutdown is only possible if the app is running. - // so we need to force kill if the app is accepted or submitted - if (!forcekill - && app.getYarnApplicationState().ordinal() < YarnApplicationState.RUNNING.ordinal()) { - log.info("Cluster {} is in a pre-running state {}. Force killing it", clustername, + if (app.getYarnApplicationState().ordinal() >= YarnApplicationState.FINISHED + .ordinal()) { + log.info("Application {} is in a terminated state {}", appName, app.getYarnApplicationState()); - forcekill = true; - } - - LaunchedApplication application = new LaunchedApplication(yarnClient, app); - applicationId = application.getApplicationId(); - - if (forcekill) { - // escalating to forced kill - application.kill("Forced stop of " + clustername + ": " + text); - } else { - try { - SliderClusterProtocol appMaster = connect(app); - Messages.StopClusterRequestProto r = - Messages.StopClusterRequestProto - .newBuilder() - .setMessage(text) - .build(); - appMaster.stopCluster(r); - - log.debug("Cluster stop command issued"); - - } catch (YarnException e) { - log.warn("Exception while trying to terminate {}", clustername, e); - return EXIT_FALSE; - } catch (IOException e) { - log.warn("Exception while trying to terminate {}", clustername, e); - return EXIT_FALSE; - } + return; } - //wait for completion. We don't currently return an exception during this process - //as the stop operation has been issued, this is just YARN. try { - if (waittime > 0) { - ApplicationReport applicationReport = - application.monitorAppToState(YarnApplicationState.FINISHED, - new Duration(waittime * 1000)); - if (applicationReport == null) { - log.info("application did not shut down in time"); - return EXIT_FALSE; - } - } - - } catch (YarnException | IOException e) { - log.warn("Exception while waiting for the application {} to shut down: {}", - clustername, e); + SliderClusterProtocol appMaster = connect(app); + Messages.StopClusterRequestProto r = + Messages.StopClusterRequestProto.newBuilder() + .setMessage(freezeArgs.message).build(); + appMaster.stopCluster(r); + log.info("Application " + appName + " is gracefully stopped."); + } catch (IOException | YarnException e){ + log.info("Failed to stop " + appName + + " gracefully, forcefully kill the app."); + yarnClient.killApplication(app.getApplicationId(), freezeArgs.message); } - - return EXIT_SUCCESS; } @Override - public int actionThaw(String clustername, ActionThawArgs thaw) throws YarnException, IOException { - validateClusterName(clustername); - verifyBindingsDefined(); + public int actionThaw(String appName, ActionThawArgs thaw) + throws YarnException, IOException { + validateClusterName(appName); // see if it is actually running and bail out; - verifyNoLiveClusters(clustername, "Start"); - - //start the cluster - return startCluster(clustername, thaw, thaw.lifetime); + verifyNoLiveClusters(appName, "Start"); + Path appDir = sliderFileSystem.buildClusterDirPath(appName); + Path appJson = new Path(appDir, appName + ".json"); + Application application = + jsonSerDeser.load(sliderFileSystem.getFileSystem(), appJson); + submitApp(application); + return 0; } - /** - * Implement flexing - * @param clustername name of the cluster - * @param roleInstances map of new role instances - * @return EXIT_SUCCESS if the #of nodes in a live cluster changed - * @throws YarnException - * @throws IOException - */ - public int flex(String clustername, Map roleInstances) + public long flex(String appName, Component component) throws YarnException, IOException { - verifyBindingsDefined(); - validateClusterName(clustername); - Path clusterDirectory = sliderFileSystem.buildClusterDirPath(clustername); - AggregateConf instanceDefinition = loadInstanceDefinitionUnresolved( - clustername, - clusterDirectory); - - ConfTreeOperations resources = - instanceDefinition.getResourceOperations(); - for (Map.Entry entry : roleInstances.entrySet()) { - String role = entry.getKey(); - String updateCountStr = entry.getValue(); - int currentCount = 0; - MapOperations component = resources.getOrAddComponent(role); - try { - // check if a relative count is specified - if (updateCountStr.startsWith("+") || updateCountStr.startsWith("-")) { - int updateCount = Integer.parseInt(updateCountStr); - // if component was specified before, get the current count - if (component.get(COMPONENT_INSTANCES) != null) { - currentCount = Integer.parseInt(component.get(COMPONENT_INSTANCES)); - if (currentCount + updateCount < 0) { - throw new BadCommandArgumentsException("The requested count " + - "of \"%s\" for role %s makes the total number of " + - "instances negative: \"%s\"", updateCount, role, - currentCount+updateCount); - } - else { - component.put(COMPONENT_INSTANCES, - Integer.toString(currentCount+updateCount)); - } - } - else { - if (updateCount < 0) { - throw new BadCommandArgumentsException("Invalid to request " + - "negative count of \"%s\" for role %s", updateCount, role); - } - else { - Map map = new HashMap<>(); - resources.confTree.components.put(role, map); - component = new MapOperations(role, map); - component.put(COMPONENT_INSTANCES, Integer.toString(updateCount)); - } - } - } - else { - int count = Integer.parseInt(updateCountStr); - resources.getOrAddComponent(role).put(COMPONENT_INSTANCES, - Integer.toString(count)); - } - } - catch (NumberFormatException e) { - throw new BadCommandArgumentsException("Requested count of role %s" + - " is not a number: \"%s\"", - role, updateCountStr); - } - - log.debug("Flexed cluster specification ( {} -> {}) : \n{}", - role, - updateCountStr, - resources); - } - SliderAMClientProvider sliderAM = new SliderAMClientProvider(getConfig()); - AbstractClientProvider provider = createClientProvider( - instanceDefinition.getInternalOperations().getGlobalOptions().getMandatoryOption( - INTERNAL_PROVIDER_NAME)); - // slider provider to validate what there is - validateInstanceDefinition(sliderAM, instanceDefinition, sliderFileSystem); - validateInstanceDefinition(provider, instanceDefinition, sliderFileSystem); - - int exitCode = EXIT_FALSE; - // save the specification - try { - InstanceIO.saveInstanceDefinition(sliderFileSystem, clusterDirectory, - instanceDefinition); - } catch (LockAcquireFailedException e) { - // lock failure - log.debug("Failed to lock dir {}", clusterDirectory, e); - log.warn("Failed to save new resource definition to {} : {}", clusterDirectory, e); - } - - // now see if it is actually running and tell it about the update if it is - ApplicationReport instance = findInstance(clustername); + validateClusterName(appName); + Path appDir = sliderFileSystem.buildClusterDirPath(appName); + Path appJson = new Path(appDir, appName + ".json"); + Application persistedApp = + jsonSerDeser.load(sliderFileSystem.getFileSystem(), appJson); + long original = 0; + for (Component persistedComp : persistedApp.getComponents()) { + if (persistedComp.getName().equals(component.getName())) { + original = persistedComp.getNumberOfContainers(); + persistedComp.setNumberOfContainers(component.getNumberOfContainers()); + } + } + jsonSerDeser + .save(sliderFileSystem.getFileSystem(), appJson, persistedApp, true); + log.info("Persisted updated component: " + component); + + ApplicationReport instance = findInstance(appName); if (instance != null) { - log.info("Flexing running cluster"); + log.info("Flexing running app " + appName); SliderClusterProtocol appMaster = connect(instance); - SliderClusterOperations clusterOps = new SliderClusterOperations(appMaster); - clusterOps.flex(instanceDefinition.getResources()); - log.info("application instance size updated"); - exitCode = EXIT_SUCCESS; + SliderClusterOperations clusterOps = + new SliderClusterOperations(appMaster); + clusterOps.flex(appName, component); + log.info("Application = " + appName + ", Component = " + component.getName() + + " size updating from " + original + " to " + component + .getNumberOfContainers()); } else { - log.info("No running instance to update"); - } - return exitCode; - } - - /** - * Validate an instance definition against a provider. - * @param provider the provider performing the validation - * @param instanceDefinition the instance definition - * @throws SliderException if invalid. - */ - protected void validateInstanceDefinition(AbstractClientProvider provider, - AggregateConf instanceDefinition, SliderFileSystem fs) throws SliderException { - try { - provider.validateInstanceDefinition(instanceDefinition, fs); - } catch (SliderException e) { - //problem, reject it - log.info("Error {} validating application instance definition ", e.getMessage()); - log.debug("Error validating application instance definition ", e); - log.info(instanceDefinition.toString()); - throw e; + String message = "Application " + appName + "does not exist in RM "; + throw new YarnException(message); } - } - - - /** - * Load the persistent cluster description - * @param clustername name of the cluster - * @return the description in the filesystem - * @throws IOException any problems loading -including a missing file - */ - @VisibleForTesting - public AggregateConf loadPersistedClusterDescription(String clustername) - throws IOException, SliderException, LockAcquireFailedException { - Path clusterDirectory = sliderFileSystem.buildClusterDirPath(clustername); - ConfPersister persister = new ConfPersister(sliderFileSystem, clusterDirectory); - AggregateConf instanceDescription = new AggregateConf(); - persister.load(instanceDescription); - return instanceDescription; - } - - /** - * Connect to a live cluster and get its current state - * @param clustername the cluster name - * @return its description - */ - @VisibleForTesting - public ClusterDescription getClusterDescription(String clustername) throws - YarnException, - IOException { - SliderClusterOperations clusterOperations = - createClusterOperations(clustername); - return clusterOperations.getClusterDescription(); + return original; } /** - * Connect to the cluster and get its current state + * Connect to a live cluster and get its current state + * + * @param appName the cluster name * @return its description */ @VisibleForTesting - public ClusterDescription getClusterDescription() throws - YarnException, - IOException { - return getClusterDescription(getDeployedClusterName()); + public Application getApplication(String appName) + throws YarnException, IOException { + validateClusterName(appName); + SliderClusterOperations clusterOperations = + createClusterOperations(appName); + return clusterOperations.getApplication(); } /** @@ -3499,14 +2451,6 @@ public ClusterDescription getClusterDescription() throws } /** - * Get the instance definition from the far end - */ - @VisibleForTesting - public AggregateConf getLiveInstanceDefinition() throws IOException, YarnException { - return createClusterOperations().getInstanceDefinition(); - } - - /** * Bond to a running cluster * @param clustername cluster name * @return the AM RPC client @@ -3515,7 +2459,6 @@ public AggregateConf getLiveInstanceDefinition() throws IOException, YarnExcepti private SliderClusterProtocol bondToCluster(String clustername) throws YarnException, IOException { - verifyBindingsDefined(); if (clustername == null) { throw unknownClusterException("(undefined)"); } @@ -3601,23 +2544,6 @@ public String toString() { return yarnClient.getApplications(); } - @VisibleForTesting - public ApplicationReport getApplicationReport(ApplicationId appId) - throws YarnException, IOException { - return new LaunchedApplication(appId, yarnClient).getApplicationReport(); - } - - /** - * The configuration used for deployment (after resolution). - * Non-null only after the client has launched the application - * @return the resolved configuration or null - */ - @VisibleForTesting - public AggregateConf getLaunchedInstanceDefinition() { - return launchedInstanceDefinition; - } - - @Override public int actionResolve(ActionResolveArgs args) throws YarnException, IOException { @@ -3793,7 +2719,7 @@ public int actionDiagnostic(ActionDiagnosticArgs diagnosticArgs) { } else if (diagnosticArgs.all) { actionDiagnosticAll(diagnosticArgs); } else if (diagnosticArgs.level) { - actionDiagnosticIntelligent(diagnosticArgs); + // agent is removed } else { // it's an unknown option log.info(CommonArgs.usage(serviceArgs, ACTION_DIAGNOSTICS)); @@ -3806,70 +2732,6 @@ public int actionDiagnostic(ActionDiagnosticArgs diagnosticArgs) { return EXIT_SUCCESS; } - private void actionDiagnosticIntelligent(ActionDiagnosticArgs diagnosticArgs) - throws YarnException, IOException, URISyntaxException { - // not using member variable clustername because we want to place - // application name after --application option and member variable - // cluster name has to be put behind action - String clusterName = diagnosticArgs.name; - requireArgumentSet(Arguments.ARG_NAME, clusterName); - - try { - validateClientConfigFile(); - log.info("Slider-client.xml is accessible"); - } catch (IOException e) { - // we are catching exceptions here because those are indication of - // validation result, and we need to print them here - log.error("validation of slider-client.xml fails because: " + e, e); - return; - } - SliderClusterOperations clusterOperations = createClusterOperations(clusterName); - // cluster not found exceptions will be thrown upstream - ClusterDescription clusterDescription = clusterOperations - .getClusterDescription(); - log.info("Slider AppMaster is accessible"); - - if (clusterDescription.state == StateValues.STATE_LIVE) { - AggregateConf instanceDefinition = clusterOperations - .getInstanceDefinition(); - String imagePath = instanceDefinition.getInternalOperations().get( - INTERNAL_APPLICATION_IMAGE_PATH); - // if null, that means slider uploaded the agent tarball for the user - // and we need to use where slider has put - if (imagePath == null) { - ApplicationReport appReport = findInstance(clusterName); - Path path1 = sliderFileSystem.getTempPathForCluster(clusterName); - if (appReport != null) { - Path subPath = new Path(path1, appReport.getApplicationId() - .toString() + "/agent"); - imagePath = subPath.toString(); - String pathStr = imagePath + "/" + AgentKeys.AGENT_TAR; - try { - validateHDFSFile(sliderFileSystem, pathStr); - log.info("Slider agent package is properly installed at " + pathStr); - } catch (FileNotFoundException e) { - log.error("can not find agent package: {}", pathStr, e); - return; - } catch (IOException e) { - log.error("can not open agent package: {}", pathStr, e); - return; - } - } - } - - String pkgTarballPath = getApplicationDefinitionPath(instanceDefinition - .getAppConfOperations()); - try { - validateHDFSFile(sliderFileSystem, pkgTarballPath); - log.info("Application package is properly installed"); - } catch (FileNotFoundException e) { - log.error("can not find application package: {}", pkgTarballPath, e); - } catch (IOException e) { - log.error("can not open application package: {} ", pkgTarballPath, e); - } - } - } - private void actionDiagnosticAll(ActionDiagnosticArgs diagnosticArgs) throws IOException, YarnException { // assign application name from param to each sub diagnostic function @@ -3950,7 +2812,7 @@ private void actionDiagnosticSlider(ActionDiagnosticArgs diagnosticArgs) if(isUnset(clusterName)){ throw new BadCommandArgumentsException("application name must be provided with --name option"); } - AggregateConf instanceDefinition = fetchInstanceDefinition(clusterName); + AggregateConf instanceDefinition = new AggregateConf(); String imagePath = instanceDefinition.getInternalOperations().get( INTERNAL_APPLICATION_IMAGE_PATH); // if null, it will be uploaded by Slider and thus at slider's path @@ -3966,21 +2828,6 @@ private void actionDiagnosticSlider(ActionDiagnosticArgs diagnosticArgs) log.info("The path of slider agent tarball on HDFS is: " + imagePath); } - private AggregateConf fetchInstanceDefinition(String clusterName) - throws YarnException, IOException { - SliderClusterOperations clusterOperations; - AggregateConf instanceDefinition = null; - try { - clusterOperations = createClusterOperations(clusterName); - instanceDefinition = clusterOperations.getInstanceDefinition(); - } catch (YarnException | IOException e) { - log.error("Failed to retrieve instance definition from YARN: " - + e.toString()); - throw e; - } - return instanceDefinition; - } - private void actionDiagnosticApplication(ActionDiagnosticArgs diagnosticArgs) throws YarnException, IOException { // not using member variable clustername because we want to place @@ -3988,7 +2835,7 @@ private void actionDiagnosticApplication(ActionDiagnosticArgs diagnosticArgs) // cluster name has to be put behind action String clusterName = diagnosticArgs.name; requireArgumentSet(Arguments.ARG_NAME, clusterName); - AggregateConf instanceDefinition = fetchInstanceDefinition(clusterName); + AggregateConf instanceDefinition = new AggregateConf(); String clusterDir = instanceDefinition.getAppConfOperations() .getGlobalOptions().get(AgentKeys.APP_ROOT); String pkgTarball = getApplicationDefinitionPath(instanceDefinition.getAppConfOperations()); @@ -4442,7 +3289,6 @@ private static void println(String message, Object ... args) { @VisibleForTesting public int actionLookup(ActionLookupArgs args) throws IOException, YarnException { - verifyBindingsDefined(); try { ApplicationId id = ConverterUtils.toApplicationId(args.id); ApplicationReport report = yarnClient.getApplicationReport(id); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java index c6cc2d0..f0201b7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java @@ -23,6 +23,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.slider.api.resource.Application; import org.apache.slider.api.types.NodeInformationList; import org.apache.slider.api.types.SliderInstanceDescription; import org.apache.slider.common.params.AbstractClusterBuildingActionArgs; @@ -61,15 +62,8 @@ * Stability: evolving */ public interface SliderClientAPI extends Service { - /** - * Destroy a cluster. There's two race conditions here - * #1 the cluster is started between verifying that there are no live - * clusters of that name. - */ - int actionDestroy(String clustername, ActionDestroyArgs destroyArgs) - throws YarnException, IOException; - int actionDestroy(String clustername) throws YarnException, + void actionDestroy(String clustername) throws YarnException, IOException; /** @@ -188,24 +182,7 @@ int actionUpdate(String clustername, */ int actionUpgrade(String clustername, ActionUpgradeArgs buildInfo) - throws YarnException, IOException; - - /** - * Get the report of a this application - * @return the app report or null if it could not be found. - * @throws IOException - * @throws YarnException - */ - ApplicationReport getApplicationReport() - throws IOException, YarnException; - - /** - * Kill the submitted application via YARN - * @throws YarnException - * @throws IOException - */ - boolean forceKillApplication(String reason) - throws YarnException, IOException; + throws YarnException, IOException; /** * Implement the list action: list all nodes @@ -214,25 +191,6 @@ boolean forceKillApplication(String reason) int actionList(String clustername, ActionListArgs args) throws IOException, YarnException; /** - * Enumerate slider instances for the current user, and the - * most recent app report, where available. - * @param listOnlyInState boolean to indicate that the instances should - * only include those in a YARN state - * minAppState <= currentState <= maxAppState - * - * @param minAppState minimum application state to include in enumeration. - * @param maxAppState maximum application state to include - * @return a map of application instance name to description - * @throws IOException Any IO problem - * @throws YarnException YARN problems - */ - Map enumSliderInstances( - boolean listOnlyInState, - YarnApplicationState minAppState, - YarnApplicationState maxAppState) - throws IOException, YarnException; - - /** * Implement the islive action: probe for a cluster of the given name existing * @return exit code */ @@ -288,7 +246,7 @@ int actionStatus(String clustername, ActionStatusArgs statusArgs) * @throws YarnException * @throws IOException */ - String actionStatus(String clustername) throws YarnException, IOException; + Application actionStatus(String clustername) throws YarnException, IOException; /** * Version Details @@ -303,7 +261,7 @@ int actionStatus(String clustername, ActionStatusArgs statusArgs) * @param freezeArgs arguments to the stop * @return EXIT_SUCCESS if the cluster was not running by the end of the operation */ - int actionFreeze(String clustername, ActionFreezeArgs freezeArgs) + void actionStop(String clustername, ActionFreezeArgs freezeArgs) throws YarnException, IOException; /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java index 258ef31..4839395 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java @@ -60,6 +60,10 @@ import java.util.Map; import java.util.Set; +//TODO, Remove this class and YarnAppListClient +// why do we need so many yarn client wrappers ? +// - yarn client already provides most of functionality already + /** * A class that extends visibility to some of the YarnClientImpl * members and data structures, and factors out pure-YARN operations @@ -69,25 +73,6 @@ protected static final Logger log = LoggerFactory.getLogger(SliderYarnClientImpl.class); /** - * Keyword to use in the {@link #emergencyForceKill(String)} - * operation to force kill all application instances belonging - * to a specific user - */ - public static final String KILL_ALL = "all"; - - @Override - protected void serviceInit(Configuration conf) throws Exception { - InetSocketAddress clientRpcAddress = SliderUtils.getRmAddress(conf); - if (!SliderUtils.isAddressDefined(clientRpcAddress)) { - // address isn't known; fail fast - throw new BindException("Invalid " + YarnConfiguration.RM_ADDRESS - + " value:" + conf.get(YarnConfiguration.RM_ADDRESS) - + " - see https://wiki.apache.org/hadoop/UnsetHostnameOrPort"); - } - super.serviceInit(conf); - } - - /** * Get the RM Client RPC interface * @return an RPC interface valid after initialization and authentication */ @@ -96,52 +81,6 @@ public ApplicationClientProtocol getRmClient() { } /** - * List Slider runninginstances belonging to a specific user. - * @deprecated use {@link #listDeployedInstances(String)} - * @param user user: "" means all users - * @return a possibly empty list of Slider AMs - */ - public List listInstances(String user) - throws YarnException, IOException { - return listDeployedInstances(user); - } - - /** - * List Slider deployedinstances belonging to a specific user. - *

- * Deployed means: known about in the YARN cluster; it will include - * any that are in the failed/finished state, as well as those queued - * for starting. - * @param user user: "" means all users - * @return a possibly empty list of Slider AMs - */ - public List listDeployedInstances(String user) - throws YarnException, IOException { - return listDeployedInstances(user, null); - } - - /** - * List Slider deployedinstances belonging to a specific user in a - * given set of states. - *

- * Deployed means: known about in the YARN cluster; it will include all apps - * in the specified set of states. - * - * @param user - * user: "" means all users - * @param appStates - * filter by a set of YarnApplicationState - * @return a possibly empty list of Slider AMs - * @throws YarnException - * @throws IOException - */ - public List listDeployedInstances(String user, - EnumSet appStates) - throws YarnException, IOException { - return listDeployedInstances(user, appStates, null); - } - - /** * List Slider deployedinstances belonging to a specific user in a * given set of states and filtered by an application name tag. *

@@ -178,21 +117,6 @@ public ApplicationClientProtocol getRmClient() { } return results; } - - /** - * find all instances of a specific app -if there is more than one in the - * YARN cluster, - * this returns them all - * @param user user; use "" for all users - * @param appname application name - * @return the list of all matching application instances - */ - public List findAllInstances(String user, String appname) - throws IOException, YarnException { - Preconditions.checkArgument(appname != null, "Null application name"); - - return listDeployedInstances(user, null, appname); - } /** * Helper method to determine if a cluster application is running -or @@ -206,122 +130,6 @@ public boolean isApplicationLive(ApplicationReport app) { return app.getYarnApplicationState().ordinal() <= YarnApplicationState.RUNNING.ordinal(); } - - /** - * Kill a running application - * @param applicationId app Id - * @param reason reason: reason for log - * @return the response - * @throws YarnException YARN problems - * @throws IOException IO problems - */ - public KillApplicationResponse killRunningApplication(ApplicationId applicationId, - String reason) - throws YarnException, IOException { - Preconditions.checkArgument(applicationId != null, "Null application Id"); - log.info("Killing application {} - {}", applicationId.getClusterTimestamp(), - reason); - KillApplicationRequest request = - Records.newRecord(KillApplicationRequest.class); - request.setApplicationId(applicationId); - return getRmClient().forceKillApplication(request); - } - - private String getUsername() throws IOException { - return UserGroupInformation.getCurrentUser().getShortUserName(); - } - - /** - * Force kill a yarn application by ID. No niceties here - * @param applicationId app Id. "all" means "kill all instances of the current user - * - */ - public void emergencyForceKill(String applicationId) - throws YarnException, IOException { - - Preconditions.checkArgument(StringUtils.isNotEmpty(applicationId), - "Null/empty application Id"); - - if (KILL_ALL.equals(applicationId)) { - // user wants all instances killed - String user = getUsername(); - log.info("Killing all applications belonging to {}", user); - Collection instances = listDeployedInstances(user, - SliderUtils.getAllLiveAppStates()); - for (ApplicationReport instance : instances) { - ApplicationId appId = instance.getApplicationId(); - log.info("Killing Application {}", appId); - killRunningApplication(appId, "forced kill"); - } - } else { - ApplicationId appId = ConverterUtils.toApplicationId(applicationId); - - log.info("Killing Application {}", applicationId); - - killRunningApplication(appId, "forced kill"); - } - } - - /** - * Monitor the submitted application for reaching the requested state. - * Will also report if the app reaches a later state (failed, killed, etc) - * Kill application if duration!= null & time expires. - * @param appId Application Id of application to be monitored - * @param duration how long to wait -must be more than 0 - * @param desiredState desired state. - * @return the application report -null on a timeout - * @throws YarnException - * @throws IOException - */ - public ApplicationReport monitorAppToState( - ApplicationId appId, YarnApplicationState desiredState, Duration duration) - throws YarnException, IOException { - - if (appId == null) { - throw new BadCommandArgumentsException("null application ID"); - } - if (duration.limit <= 0) { - throw new BadCommandArgumentsException("Invalid monitoring duration"); - } - log.debug("Waiting {} millis for app to reach state {} ", - duration.limit, - desiredState); - duration.start(); - try { - while (true) { - // Get application report for the appId we are interested in - - ApplicationReport r = getApplicationReport(appId); - - log.debug("queried status is\n{}", - new SliderUtils.OnDemandReportStringifier(r)); - - YarnApplicationState state = r.getYarnApplicationState(); - if (state.ordinal() >= desiredState.ordinal()) { - log.debug("App in desired state (or higher) :{}", state); - return r; - } - if (duration.getLimitExceeded()) { - log.debug( - "Wait limit of {} millis to get to state {}, exceeded; app status\n {}", - duration.limit, - desiredState, - new SliderUtils.OnDemandReportStringifier(r)); - return null; - } - - // sleep 1s. - try { - Thread.sleep(1000); - } catch (InterruptedException ignored) { - log.debug("Thread sleep in monitoring loop interrupted"); - } - } - } finally { - duration.close(); - } - } - /** * find all live instances of a specific app -if there is >1 in the cluster, * this returns them all. State should be running or less diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ipc/SliderApplicationIpcClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ipc/SliderApplicationIpcClient.java index a007326..3b5147f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ipc/SliderApplicationIpcClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ipc/SliderApplicationIpcClient.java @@ -112,17 +112,6 @@ public ConfTreeOperations getDesiredResources() throws IOException { } } - - @Override - public void putDesiredResources(ConfTree updated) throws IOException { - try { - operations.flex(updated); - } catch (IOException e) { - throw convert(e); - } - } - - @Override public AggregateConf getResolvedModel() throws IOException { try { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java index 702233a..f3cc8e3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java @@ -25,6 +25,8 @@ import org.apache.slider.api.SliderClusterProtocol; import org.apache.slider.api.StateValues; import org.apache.slider.api.proto.Messages; +import org.apache.slider.api.resource.Application; +import org.apache.slider.api.resource.Component; import org.apache.slider.api.types.ApplicationLivenessInformation; import org.apache.slider.api.types.ComponentInformation; import org.apache.slider.api.types.ContainerInformation; @@ -39,6 +41,7 @@ import org.apache.slider.core.exceptions.SliderException; import org.apache.slider.core.exceptions.WaitTimeoutException; import org.apache.slider.core.persist.ConfTreeSerDeser; +import org.apache.slider.core.persist.JsonSerDeser; import org.codehaus.jackson.JsonParseException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -61,6 +64,8 @@ log = LoggerFactory.getLogger(SliderClusterOperations.class); private final SliderClusterProtocol appMaster; + private static final JsonSerDeser jsonSerDeser = + new JsonSerDeser(Application.class); private static final Messages.EmptyPayloadProto EMPTY; static { EMPTY = Messages.EmptyPayloadProto.newBuilder().build(); @@ -130,49 +135,21 @@ public String echo(String text) throws YarnException, IOException { * Connect to a live cluster and get its current state * @return its description */ - public ClusterDescription getClusterDescription() - throws YarnException, IOException { - + public Application getApplication() throws YarnException, IOException { Messages.GetJSONClusterStatusRequestProto req = Messages.GetJSONClusterStatusRequestProto.newBuilder().build(); Messages.GetJSONClusterStatusResponseProto resp = appMaster.getJSONClusterStatus(req); String statusJson = resp.getClusterSpec(); try { - return ClusterDescription.fromJson(statusJson); + return jsonSerDeser.fromJson(statusJson); } catch (JsonParseException e) { - log.error("Exception " + e + " parsing:\n" + statusJson, e); + log.error("Error when parsing app json file", e); throw e; } } /** - * Get the AM instance definition. - *

- * See {@link SliderClusterProtocol#getInstanceDefinition(Messages.GetInstanceDefinitionRequestProto)} - * @return current slider AM aggregate definition - * @throws YarnException - * @throws IOException - */ - public AggregateConf getInstanceDefinition() - throws YarnException, IOException { - Messages.GetInstanceDefinitionRequestProto.Builder builder = - Messages.GetInstanceDefinitionRequestProto.newBuilder(); - - Messages.GetInstanceDefinitionRequestProto request = builder.build(); - Messages.GetInstanceDefinitionResponseProto response = - appMaster.getInstanceDefinition(request); - - ConfTreeSerDeser confTreeSerDeser = new ConfTreeSerDeser(); - - ConfTree internal = confTreeSerDeser.fromJson(response.getInternal()); - ConfTree resources = confTreeSerDeser.fromJson(response.getResources()); - ConfTree app = confTreeSerDeser.fromJson(response.getApplication()); - AggregateConf instanceDefinition = - new AggregateConf(resources, app, internal); - return instanceDefinition; - } - /** * Kill a container * @param id container ID * @return a success flag @@ -315,22 +292,14 @@ public int waitForRoleInstanceLive(String role, long timeout) return state; } - /** - * Flex operation - * @param resources new resources - * @return the response - * @throws IOException - */ - public boolean flex(ConfTree resources) throws IOException { - Messages.FlexClusterRequestProto request = - Messages.FlexClusterRequestProto.newBuilder() - .setClusterSpec(resources.toJson()) - .build(); - Messages.FlexClusterResponseProto response = appMaster.flexCluster(request); - return response.getResponse(); + public void flex(String appName, Component component) throws IOException{ + Messages.FlexComponentRequestProto request = + Messages.FlexComponentRequestProto.newBuilder() + .setNumberOfContainers(component.getNumberOfContainers().intValue()) + .setName(appName).build(); + appMaster.flexComponent(request); } - /** * Commit (possibly delayed) AM suicide * diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationApiRestClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationApiRestClient.java index 573ef64..4c376e0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationApiRestClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationApiRestClient.java @@ -177,29 +177,6 @@ public ConfTreeOperations getDesiredResources() throws IOException { } @Override - public void putDesiredResources(ConfTree updated) throws IOException { - WebResource resource = applicationResource(MODEL_DESIRED_RESOURCES); - try { - - // put operation. The result is discarded; it does help validate - // that the operation returned a JSON data structure as well as a 200 - // response. - - resource.accept(MediaType.APPLICATION_JSON_TYPE) - .type(MediaType.APPLICATION_JSON_TYPE) - .entity(updated) - .put(ConfTree.class); - } catch (ClientHandlerException ex) { - throw ExceptionConverter.convertJerseyException("PUT", - resource.getURI().toString(), - ex); - } catch (UniformInterfaceException ex) { - throw ExceptionConverter.convertJerseyException("PUT", - resource.getURI().toString(), ex); - } - } - - @Override public AggregateConf getResolvedModel() throws IOException { return getApplicationResource(MODEL_RESOLVED, AggregateConf.class); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java index 4bf1b5b..e10ca13 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java @@ -18,6 +18,8 @@ package org.apache.slider.common; +import org.apache.hadoop.yarn.api.ApplicationConstants; + import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -238,7 +240,7 @@ String STDERR_AM = "slider-err.txt"; String DEFAULT_GC_OPTS = ""; - String HADOOP_USER_NAME = "HADOOP_USER_NAME"; + String HADOOP_USER_NAME = ApplicationConstants.Environment.USER.toString(); String HADOOP_PROXY_USER = "HADOOP_PROXY_USER"; String SLIDER_PASSPHRASE = "SLIDER_PASSPHRASE"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java index 72dd44f..b666834 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java @@ -73,6 +73,9 @@ int DEFAULT_YARN_QUEUE_PRIORITY = 1; + String KEY_AM_RESOURCE_MEM = "slider.am.resource.memory"; + long DEFAULT_KEY_AM_RESOURCE_MEM = 1024; + /** * The slider base path: {@value} * Defaults to HomeDir/.slider diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java index 20c7831..82dbd3b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java @@ -1221,6 +1221,29 @@ public static boolean isPortAvailable(int port) { return buildEnvMap(roleOpts, null); } + + // Build env map: key -> value; + // value will be replaced by the corresponding value in tokenMap, if any. + public static Map buildEnvMap( + org.apache.slider.api.resource.Configuration conf, + Map tokenMap) { + if (tokenMap == null) { + return conf.getEnv(); + } + Map env = new HashMap<>(); + for (Map.Entry entry : conf.getEnv().entrySet()) { + String key = entry.getKey(); + String val = entry.getValue(); + for (Map.Entry token : tokenMap.entrySet()) { + val = val.replaceAll(Pattern.quote(token.getKey()), + token.getValue()); + } + env.put(key,val); + } + return env; + } + + public static Map buildEnvMap(Map roleOpts, Map tokenMap) { Map env = new HashMap<>(); @@ -1273,8 +1296,8 @@ public static void applyCommandLineRoleOptsToRoleMap( * @param clustername cluster name * @throws BadCommandArgumentsException if it is invalid */ - public static void validateClusterName(String clustername) throws - BadCommandArgumentsException { + public static void validateClusterName(String clustername) + throws BadCommandArgumentsException { if (!isClusternameValid(clustername)) { throw new BadCommandArgumentsException( "Illegal cluster name: " + clustername); @@ -1603,14 +1626,12 @@ public static String sequenceToString(CharSequence charSequence) { * @param sliderConfDir relative path to the dir containing slider config * options to put on the classpath -or null * @param libdir directory containing the JAR files - * @param config the configuration * @param usingMiniMRCluster flag to indicate the MiniMR cluster is in use * (and hence the current classpath should be used, not anything built up) * @return a classpath */ public static ClasspathConstructor buildClasspath(String sliderConfDir, String libdir, - Configuration config, SliderFileSystem sliderFileSystem, boolean usingMiniMRCluster) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java deleted file mode 100644 index 7190c3a..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java +++ /dev/null @@ -1,233 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.core.launch; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.security.Credentials; -import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; -import org.apache.hadoop.yarn.api.records.Priority; -import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.client.api.YarnClientApplication; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.util.Records; -import org.apache.slider.client.SliderYarnClientImpl; -import org.apache.slider.common.tools.CoreFileSystem; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.Map; -import java.util.Set; - -public class AppMasterLauncher extends AbstractLauncher { - - - private static final Logger log = - LoggerFactory.getLogger(AppMasterLauncher.class); - - public final YarnClientApplication application; - public final String name; - public final String type; - public final ApplicationSubmissionContext submissionContext; - public final ApplicationId appId; - public final boolean secureCluster; - private int maxAppAttempts = 0; - private boolean keepContainersOverRestarts = true; - private String queue = YarnConfiguration.DEFAULT_QUEUE_NAME; - private int priority = 1; - private final Resource resource = Records.newRecord(Resource.class); - private final SliderYarnClientImpl yarnClient; - private Long submitTime; - - /** - * Build the AM Launcher - * @param name app name - * @param type application type - * @param conf hadoop config - * @param fs filesystem binding - * @param yarnClient yarn client - * @param secureCluster flag to indicate secure cluster - * @param options map of options. All values are extracted in this constructor only - * @param resourceGlobalOptions global options - * @param applicationTags any app tags - * @param credentials initial set of credentials - * @throws IOException - * @throws YarnException - */ - public AppMasterLauncher(String name, - String type, - Configuration conf, - CoreFileSystem fs, - SliderYarnClientImpl yarnClient, - boolean secureCluster, - Map options, - Map resourceGlobalOptions, - Set applicationTags, - Credentials credentials) throws IOException, YarnException { - super(conf, fs, credentials); - this.yarnClient = yarnClient; - this.application = yarnClient.createApplication(); - this.name = name; - this.type = type; - this.secureCluster = secureCluster; - - submissionContext = application.getApplicationSubmissionContext(); - appId = submissionContext.getApplicationId(); - // set the application name; - submissionContext.setApplicationName(name); - // app type used in service enum; - submissionContext.setApplicationType(type); - if (!applicationTags.isEmpty()) { - submissionContext.setApplicationTags(applicationTags); - } - submissionContext.setNodeLabelExpression(extractLabelExpression(options)); - - extractAmRetryCount(submissionContext, resourceGlobalOptions); - extractResourceRequirements(resource, options); - extractLogAggregationContext(resourceGlobalOptions); - } - - public void setMaxAppAttempts(int maxAppAttempts) { - this.maxAppAttempts = maxAppAttempts; - } - - public void setKeepContainersOverRestarts(boolean keepContainersOverRestarts) { - this.keepContainersOverRestarts = keepContainersOverRestarts; - } - - - public Resource getResource() { - return resource; - } - - public void setMemory(int memory) { - resource.setMemory(memory); - } - - public void setVirtualCores(int cores) { - resource.setVirtualCores(cores); - } - - public ApplicationId getApplicationId() { - return appId; - } - - public int getMaxAppAttempts() { - return maxAppAttempts; - } - - public boolean isKeepContainersOverRestarts() { - return keepContainersOverRestarts; - } - - public String getQueue() { - return queue; - } - - public int getPriority() { - return priority; - } - - public void setQueue(String queue) { - this.queue = queue; - } - - public void setPriority(int priority) { - this.priority = priority; - } - - /** - * Complete the launch context (copy in env vars, etc). - * @return the container to launch - */ - public ApplicationSubmissionContext completeAppMasterLaunch() - throws IOException { - - //queue priority - Priority pri = Records.newRecord(Priority.class); - pri.setPriority(priority); - submissionContext.setPriority(pri); - - // Set the queue to which this application is to be submitted in the RM - // Queue for App master - - submissionContext.setQueue(queue); - - - //container requirements - submissionContext.setResource(resource); - submissionContext.setLogAggregationContext(logAggregationContext); - - if (keepContainersOverRestarts) { - log.debug("Requesting cluster stays running over AM failure"); - submissionContext.setKeepContainersAcrossApplicationAttempts(true); - } - - if (maxAppAttempts > 0) { - log.debug("Setting max AM attempts to {}", maxAppAttempts); - submissionContext.setMaxAppAttempts(maxAppAttempts); - } - - if (secureCluster) { - //tokens - log.debug("Credentials: {}", - CredentialUtils.dumpTokens(getCredentials(), "\n")); - - } else { - propagateUsernameInInsecureCluster(); - } - completeContainerLaunch(); - submissionContext.setAMContainerSpec(containerLaunchContext); - return submissionContext; - } - - /** - * Submit the application. - * @return a launched application representing the submitted application - * @throws IOException - * @throws YarnException - */ - public LaunchedApplication submitApplication() throws IOException, YarnException { - completeAppMasterLaunch(); - log.info("Submitting application to Resource Manager"); - ApplicationId applicationId = - yarnClient.submitApplication(submissionContext); - // implicit success; record the time - submitTime = System.currentTimeMillis(); - return new LaunchedApplication(applicationId, yarnClient); - } - - /** - * Build a serializable application report. This is a very minimal - * report that contains the application Id, name and type —the information - * available - * @return a data structure which can be persisted - */ - public SerializedApplicationReport createSerializedApplicationReport() { - SerializedApplicationReport sar = new SerializedApplicationReport(); - sar.applicationId = appId.toString(); - sar.name = name; - sar.applicationType = type; - sar.queue = queue; - sar.submitTime = submitTime; - return sar; - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/LaunchedApplication.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/LaunchedApplication.java deleted file mode 100644 index 632e3fd..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/LaunchedApplication.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.core.launch; - -import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse; -import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.ApplicationReport; -import org.apache.hadoop.yarn.api.records.YarnApplicationState; -import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.slider.client.SliderYarnClientImpl; -import org.apache.slider.common.tools.Duration; - -import java.io.IOException; - -/** - * Launched App with logic around it. - */ -public class LaunchedApplication { - - protected final ApplicationId applicationId; - protected final SliderYarnClientImpl yarnClient; - - public LaunchedApplication(ApplicationId applicationId, - SliderYarnClientImpl yarnClient) { - assert applicationId != null; - assert yarnClient != null; - this.applicationId = applicationId; - this.yarnClient = yarnClient; - } - - public LaunchedApplication(SliderYarnClientImpl yarnClient, - ApplicationReport report) { - this.yarnClient = yarnClient; - this.applicationId = report.getApplicationId(); - } - - public ApplicationId getApplicationId() { - return applicationId; - } - - /** - * Monitor the submitted application for reaching the requested state. - * Will also report if the app reaches a later state (failed, killed, etc) - * Kill application if duration!= null & time expires. - * @param duration how long to wait -must be more than 0 - * @param desiredState desired state. - * @return the application report -null on a timeout - * @throws YarnException - * @throws IOException - */ - public ApplicationReport monitorAppToState(YarnApplicationState desiredState, Duration duration) - throws YarnException, IOException { - return yarnClient.monitorAppToState(applicationId, desiredState, duration); - } - - /** - * Kill the submitted application by sending a call to the ASM - * @throws YarnException - * @throws IOException - */ - public boolean forceKill(String reason) - throws YarnException, IOException { - if (applicationId != null) { - yarnClient.killRunningApplication(applicationId, reason); - return true; - } - return false; - } - - /** - * Kill the application - * @return the response - * @throws YarnException YARN problems - * @throws IOException IO problems - */ - public KillApplicationResponse kill(String reason) throws - YarnException, - IOException { - return yarnClient.killRunningApplication(applicationId, reason); - } - - /** - * Get the application report of this application - * @return an application report - * @throws YarnException - * @throws IOException - */ - public ApplicationReport getApplicationReport() - throws YarnException, IOException { - return yarnClient.getApplicationReport(applicationId); - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/RunningApplication.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/RunningApplication.java deleted file mode 100644 index 14c522c..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/RunningApplication.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.core.launch; - -import org.apache.hadoop.yarn.api.records.ApplicationReport; -import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.slider.api.SliderClusterProtocol; -import org.apache.slider.client.SliderYarnClientImpl; -import org.apache.slider.common.SliderExitCodes; -import org.apache.slider.core.exceptions.SliderException; -import org.apache.slider.server.appmaster.rpc.RpcBinder; - -import java.io.IOException; - -import static org.apache.slider.common.Constants.CONNECT_TIMEOUT; -import static org.apache.slider.common.Constants.RPC_TIMEOUT; - -/** - * A running application built from an app report. This one - * can be talked to - */ -public class RunningApplication extends LaunchedApplication { - - private final ApplicationReport applicationReport; - public RunningApplication(SliderYarnClientImpl yarnClient, - ApplicationReport applicationReport) { - super(yarnClient, applicationReport); - this.applicationReport = applicationReport; - } - - public ApplicationReport getApplicationReport() { - return applicationReport; - } - - - /** - * Connect to a Slider AM - * @param app application report providing the details on the application - * @return an instance - * @throws YarnException - * @throws IOException - */ - public SliderClusterProtocol connect(ApplicationReport app) throws - YarnException, - IOException { - - try { - return RpcBinder.getProxy(yarnClient.getConfig(), - yarnClient.getRmClient(), - app, - CONNECT_TIMEOUT, - RPC_TIMEOUT); - } catch (InterruptedException e) { - throw new SliderException(SliderExitCodes.EXIT_TIMED_OUT, - e, - "Interrupted waiting for communications with the Application Master"); - } - } - -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java index 01444fd..42e103a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java @@ -103,19 +103,6 @@ public void validateInstanceDefinition(AggregateConf instanceDefinition, SliderF /** - * Any provider-side alteration of a configuration can take place here. - * @param aggregateConf config to patch - * @throws IOException IO problems - * @throws SliderException Slider-specific issues - */ - public void prepareInstanceConfiguration(AggregateConf aggregateConf) throws - SliderException, - IOException { - //default: do nothing - } - - - /** * Prepare the AM settings for launch * @param fileSystem filesystem * @param serviceConf configuration of the client @@ -234,7 +221,7 @@ public void preflightValidateClusterConfiguration(SliderFileSystem sliderFileSys * @param appDescription brief description of the application * @return */ - public final Set createApplicationTags(String appName, + public static final Set createApplicationTags(String appName, String appVersion, String appDescription) { Set tags = new HashSet<>(); tags.add(SliderUtils.createNameTag(appName)); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java index 41b26e9..e879ac2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java @@ -76,7 +76,6 @@ private static final Logger log = LoggerFactory.getLogger(AbstractProviderService.class); protected StateAccessForProviders amState; - protected URL amWebAPI; protected YarnRegistryViewForProviders yarnRegistry; protected QueueAccess queueAccess; @@ -129,68 +128,6 @@ public void notifyContainerCompleted(ContainerId containerId) { } /** - * Load default Configuration - * @param confDir configuration directory - * @return configuration - * @throws BadCommandArgumentsException - * @throws IOException - */ - @Override - public Configuration loadProviderConfigurationInformation(File confDir) - throws BadCommandArgumentsException, IOException { - return new Configuration(false); - } - - /** - * Load a specific XML configuration file for the provider config - * @param confDir configuration directory - * @param siteXMLFilename provider-specific filename - * @return a configuration to be included in status - * @throws BadCommandArgumentsException argument problems - * @throws IOException IO problems - */ - protected Configuration loadProviderConfigurationInformation(File confDir, - String siteXMLFilename) - throws BadCommandArgumentsException, IOException { - Configuration siteConf; - File siteXML = new File(confDir, siteXMLFilename); - if (!siteXML.exists()) { - throw new BadCommandArgumentsException( - "Configuration directory %s doesn't contain %s - listing is %s", - confDir, siteXMLFilename, SliderUtils.listDir(confDir)); - } - - //now read it in - siteConf = ConfigHelper.loadConfFromFile(siteXML); - log.info("{} file is at {}", siteXMLFilename, siteXML); - log.info(ConfigHelper.dumpConfigToString(siteConf)); - return siteConf; - } - - /** - * No-op implementation of this method. - */ - @Override - public void initializeApplicationConfiguration( - AggregateConf instanceDefinition, SliderFileSystem fileSystem, - String roleGroup) - throws IOException, SliderException { - } - - /** - * No-op implementation of this method. - * - * {@inheritDoc} - */ - @Override - public void validateApplicationConfiguration(AggregateConf instance, - File confDir, - boolean secure) - throws IOException, SliderException { - - } - - /** * Scan through the roles and see if it is supported. * @param role role to look for * @return true if the role is known about -and therefore @@ -374,7 +311,6 @@ public void buildEndpointDetails(Map details) { public void applyInitialRegistryDefinitions(URL amWebURI, ServiceRecord serviceRecord) throws IOException { - this.amWebAPI = amWebURI; } /** @@ -422,13 +358,6 @@ public void execute(List operations) { operation.execute(this); } } - /** - * No-op implementation of this method. - */ - @Override - public void rebuildContainerDetails(List liveContainers, - String applicationId, Map providerRoles) { - } @Override public boolean processContainerStatus(ContainerId containerId, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderCore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderCore.java index 9767430..b07fc29 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderCore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderCore.java @@ -19,9 +19,6 @@ package org.apache.slider.providers; import org.apache.hadoop.conf.Configuration; -import org.apache.slider.core.conf.AggregateConf; -import org.apache.slider.core.conf.ConfTree; -import org.apache.slider.core.exceptions.SliderException; import java.util.List; public interface ProviderCore { @@ -31,13 +28,4 @@ List getRoles(); Configuration getConf(); - - /** - * Verify that an instance definition is considered valid by the provider - * @param instanceDefinition instance definition - * @throws SliderException if the configuration is not valid - */ - void validateInstanceDefinition(AggregateConf instanceDefinition) throws - SliderException; - } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java index 761ac0f..e0299e7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java @@ -18,7 +18,7 @@ package org.apache.slider.providers; -import org.apache.slider.api.ResourceKeys; +import org.apache.slider.api.resource.Component; /** * Provider role and key for use in app requests. @@ -34,16 +34,8 @@ public int nodeFailureThreshold; public final long placementTimeoutSeconds; public final String labelExpression; + public final Component component; - public ProviderRole(String name, int id) { - this(name, - name, - id, - PlacementPolicy.DEFAULT, - ResourceKeys.DEFAULT_NODE_FAILURE_THRESHOLD, - ResourceKeys.DEFAULT_PLACEMENT_ESCALATE_DELAY_SECONDS, - ResourceKeys.DEF_YARN_LABEL_EXPRESSION); - } /** * Create a provider role @@ -67,7 +59,7 @@ public ProviderRole(String name, policy, nodeFailureThreshold, placementTimeoutSeconds, - labelExpression); + labelExpression, null); } /** @@ -87,7 +79,8 @@ public ProviderRole(String name, int policy, int nodeFailureThreshold, long placementTimeoutSeconds, - String labelExpression) { + String labelExpression, + Component component) { this.name = name; if (group == null) { this.group = name; @@ -99,6 +92,8 @@ public ProviderRole(String name, this.nodeFailureThreshold = nodeFailureThreshold; this.placementTimeoutSeconds = placementTimeoutSeconds; this.labelExpression = labelExpression; + this.component = component; + } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java index 4ca9326..e6091d3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java @@ -18,18 +18,15 @@ package org.apache.slider.providers; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; import org.apache.hadoop.registry.client.types.ServiceRecord; import org.apache.hadoop.service.Service; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.slider.api.ClusterDescription; +import org.apache.slider.api.resource.Application; import org.apache.slider.common.tools.SliderFileSystem; import org.apache.slider.core.conf.AggregateConf; -import org.apache.slider.core.conf.MapOperations; -import org.apache.slider.core.exceptions.BadCommandArgumentsException; import org.apache.slider.core.exceptions.SliderException; import org.apache.slider.core.launch.ContainerLauncher; import org.apache.slider.core.main.ExitCodeProvider; @@ -52,26 +49,10 @@ /** * Set up the entire container launch context - * @param containerLauncher - * @param instanceDefinition - * @param container - * @param providerRole - * @param sliderFileSystem - * @param generatedConfPath - * @param appComponent - * @param containerTmpDirPath */ void buildContainerLaunchContext(ContainerLauncher containerLauncher, - AggregateConf instanceDefinition, - Container container, - ProviderRole providerRole, - SliderFileSystem sliderFileSystem, - Path generatedConfPath, - MapOperations resourceComponent, - MapOperations appComponent, - Path containerTmpDirPath) throws - IOException, - SliderException; + Application application, Container container, ProviderRole providerRole, + SliderFileSystem sliderFileSystem) throws IOException, SliderException; /** * Notify the providers of container completion @@ -103,44 +84,6 @@ boolean exec(AggregateConf instanceDefinition, */ boolean isSupportedRole(String role); - /** - * Load a specific XML configuration file for the provider config - * @param confDir configuration directory - * @return a configuration to be included in status - * @throws BadCommandArgumentsException - * @throws IOException - */ - Configuration loadProviderConfigurationInformation(File confDir) - throws BadCommandArgumentsException, IOException; - - /** - * The application configuration should be initialized here - * - * @param instanceDefinition - * @param fileSystem - * @param roleGroup - * @throws IOException - * @throws SliderException - */ - void initializeApplicationConfiguration(AggregateConf instanceDefinition, - SliderFileSystem fileSystem, String roleGroup) throws IOException, - SliderException; - - /** - * This is a validation of the application configuration on the AM. - * Here is where things like the existence of keytabs and other - * not-seen-client-side properties can be tested, before - * the actual process is spawned. - * @param instanceDefinition clusterSpecification - * @param confDir configuration directory - * @param secure flag to indicate that secure mode checks must exist - * @throws IOException IO problemsn - * @throws SliderException any failure - */ - void validateApplicationConfiguration(AggregateConf instanceDefinition, - File confDir, - boolean secure - ) throws IOException, SliderException; /* * Build the provider status, can be empty @@ -197,17 +140,6 @@ void applyInitialRegistryDefinitions(URL amWebURI, ContainerReleaseSelector createContainerReleaseSelector(); /** - * On AM restart (for whatever reason) this API is required to rebuild the AM - * internal state with the containers which were already assigned and running - * - * @param liveContainers - * @param applicationId - * @param providerRoles - */ - void rebuildContainerDetails(List liveContainers, - String applicationId, Map providerRoles); - - /** * Process container status * @return true if status needs to be requested again, false otherwise */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java index f33db9b..804cd1b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java @@ -18,7 +18,6 @@ package org.apache.slider.providers; -import org.apache.commons.io.FileUtils; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; @@ -37,24 +36,21 @@ import org.apache.slider.api.OptionKeys; import org.apache.slider.api.ResourceKeys; import org.apache.slider.api.RoleKeys; +import org.apache.slider.api.resource.Application; +import org.apache.slider.api.resource.Component; +import org.apache.slider.api.resource.ConfigFile; +import org.apache.slider.api.resource.Configuration; import org.apache.slider.common.SliderKeys; import org.apache.slider.common.SliderXmlConfKeys; import org.apache.slider.common.tools.SliderFileSystem; import org.apache.slider.common.tools.SliderUtils; -import org.apache.slider.core.conf.AggregateConf; -import org.apache.slider.core.conf.ConfTreeOperations; -import org.apache.slider.core.conf.MapOperations; import org.apache.slider.core.exceptions.BadCommandArgumentsException; -import org.apache.slider.core.exceptions.BadConfigException; import org.apache.slider.core.exceptions.NoSuchNodeException; import org.apache.slider.core.exceptions.SliderException; import org.apache.slider.core.launch.ContainerLauncher; import org.apache.slider.core.registry.docstore.ConfigFormat; -import org.apache.slider.core.registry.docstore.ConfigUtils; -import org.apache.slider.core.registry.docstore.ExportEntry; import org.apache.slider.core.registry.docstore.PublishedConfiguration; import org.apache.slider.core.registry.docstore.PublishedConfigurationOutputter; -import org.apache.slider.core.registry.docstore.PublishedExports; import org.apache.slider.server.appmaster.state.RoleInstance; import org.apache.slider.server.appmaster.state.StateAccessForProviders; import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders; @@ -66,16 +62,10 @@ import java.net.URI; import java.util.ArrayList; import java.util.Collection; -import java.util.Date; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; -import java.util.TreeMap; -import java.util.TreeSet; import java.util.regex.Pattern; /** @@ -138,32 +128,6 @@ public static boolean addProviderJar( } } } - - /** - * Add/overwrite the agent tarball (overwritten every time application is - * restarted). - * @param provider an instance of a provider class - * @param tarName name of the tarball to upload - * @param sliderFileSystem the file system - * @param agentDir directory to upload to - * @return true the location could be determined and the file added - * @throws IOException if the upload fails - */ - public static boolean addAgentTar(Object provider, - String tarName, - SliderFileSystem sliderFileSystem, - Path agentDir) throws - IOException { - File localFile = SliderUtils.findContainingJar(provider.getClass()); - if(localFile != null) { - String parentDir = localFile.getParent(); - Path agentTarPath = new Path(parentDir, tarName); - sliderFileSystem.getFileSystem().copyFromLocalFile(false, true, - agentTarPath, agentDir); - return true; - } - return false; - } /** * Loads all dependency jars from the default path. @@ -193,132 +157,24 @@ public static void addAllDependencyJars( libDir, libLocalSrcDir); } - - /** - * Validate the requested number of instances of a component. - *

- * If max <= 0: min <= count - * If max > 0: min <= count <= max - * @param instanceDescription configuration - * @param name node class name - * @param min requested heap size - * @param max maximum value. - * @throws BadCommandArgumentsException if the values are out of range - */ - public void validateNodeCount(AggregateConf instanceDescription, - String name, int min, int max) - throws BadCommandArgumentsException { - MapOperations component = - instanceDescription.getResourceOperations().getComponent(name); - int count; - if (component == null) { - count = 0; - } else { - count = component.getOptionInt(ResourceKeys.COMPONENT_INSTANCES, 0); - } - validateNodeCount(name, count, min, max); - } - - /** - * Validate the count is between min and max. - *

- * If max <= 0: min <= count - * If max > 0: min <= count <= max - * @param name node class name - * @param count requested node count - * @param min requested heap size - * @param max maximum value. - * @throws BadCommandArgumentsException if the values are out of range - */ - public void validateNodeCount(String name, - int count, - int min, - int max) throws BadCommandArgumentsException { - if (count < min) { - throw new BadCommandArgumentsException( - "requested no of %s nodes: %d is below the minimum of %d", name, count, - min); - } - if (max > 0 && count > max) { - throw new BadCommandArgumentsException( - "requested no of %s nodes: %d is above the maximum of %d", name, count, - max); - } - } - - /** - * Copy options beginning with "site.configName." prefix from options map - * to sitexml map, removing the prefix and substituting the tokens - * specified in the tokenMap. - * @param options source map - * @param sitexml destination map - * @param configName optional ".configName" portion of the prefix - * @param tokenMap key/value pairs to substitute into the option values - */ - public void propagateSiteOptions(Map options, - Map sitexml, - String configName, - Map tokenMap) { - String prefix = OptionKeys.SITE_XML_PREFIX + - (!configName.isEmpty() ? configName + "." : ""); - propagateOptions(options, sitexml, tokenMap, prefix); - } - - /** - * Copy options beginning with prefix from options map - * to sitexml map, removing the prefix and substituting the tokens - * specified in the tokenMap. - * @param options source map - * @param sitexml destination map - * @param tokenMap key/value pairs to substitute into the option values - * @param prefix which options to copy to destination map - */ - public void propagateOptions(Map options, - Map sitexml, - Map tokenMap, - String prefix) { - for (Map.Entry entry : options.entrySet()) { - String key = entry.getKey(); - if (key.startsWith(prefix)) { - String envName = key.substring(prefix.length()); - if (!envName.isEmpty()) { - String value = entry.getValue(); - if (tokenMap != null) { - for (Map.Entry token : tokenMap.entrySet()) { - value = value.replaceAll(Pattern.quote(token.getKey()), - token.getValue()); - } - } - sitexml.put(envName, value); - } - } - } - } - - /** - * Substitute tokens into option map values, returning a new map. - * @param options source map - * @param tokenMap key/value pairs to substitute into the option values - * @return map with substituted values - */ - public Map filterSiteOptions(Map options, + // Build key -> value map + // value will be substituted by corresponding data in tokenMap + public Map substituteConfigs(Map configs, Map tokenMap) { - String prefix = OptionKeys.SITE_XML_PREFIX; String format = "${%s}"; Map filteredOptions = new HashMap<>(); - for (Map.Entry entry : options.entrySet()) { + for (Map.Entry entry : configs.entrySet()) { String key = entry.getKey(); - if (key.startsWith(prefix)) { - String value = entry.getValue(); - if (tokenMap != null) { - for (Map.Entry token : tokenMap.entrySet()) { - value = value.replaceAll(Pattern.quote(token.getKey()), - token.getValue()); - } + String value = entry.getValue(); + if (tokenMap != null) { + for (Map.Entry token : tokenMap.entrySet()) { + value = + value.replaceAll(Pattern.quote(token.getKey()), token.getValue()); } - filteredOptions.put(String.format(format, key), value); } + filteredOptions.put(String.format(format, key), value); } + return filteredOptions; } @@ -345,28 +201,27 @@ public int getRoleResourceRequirement(String val, return intVal; } + /** * Localize the service keytabs for the application. * @param launcher container launcher - * @param instanceDefinition app specification * @param fileSystem file system - * @param clusterName app name * @throws IOException trouble uploading to HDFS */ public void localizeServiceKeytabs(ContainerLauncher launcher, - AggregateConf instanceDefinition, SliderFileSystem fileSystem, - String clusterName) throws IOException { - ConfTreeOperations appConf = instanceDefinition.getAppConfOperations(); - String keytabPathOnHost = appConf.getComponent(COMPONENT_AM).get( - SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH); + SliderFileSystem fileSystem, Application application) throws IOException { + + Configuration conf = application.getConfiguration(); + String keytabPathOnHost = + conf.getProperty(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH); if (SliderUtils.isUnset(keytabPathOnHost)) { - String amKeytabName = appConf.getComponent(COMPONENT_AM).get( - SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME); - String keytabDir = appConf.getComponent(COMPONENT_AM).get( - SliderXmlConfKeys.KEY_HDFS_KEYTAB_DIR); + String amKeytabName = + conf.getProperty(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME); + String keytabDir = + conf.getProperty(SliderXmlConfKeys.KEY_HDFS_KEYTAB_DIR); // we need to localize the keytab files in the directory Path keytabDirPath = fileSystem.buildKeytabPath(keytabDir, null, - clusterName); + application.getName()); boolean serviceKeytabsDeployed = false; if (fileSystem.getFileSystem().exists(keytabDirPath)) { FileStatus[] keytabs = fileSystem.getFileSystem().listStatus( @@ -395,591 +250,115 @@ public void localizeServiceKeytabs(ContainerLauncher launcher, } - /** - * Upload a local file to the cluster security dir in HDFS. If the file - * already exists, it is not replaced. - * @param resource file to upload - * @param fileSystem file system - * @param clusterName app name - * @return Path of the uploaded file - * @throws IOException file cannot be uploaded - */ - private Path uploadSecurityResource(File resource, - SliderFileSystem fileSystem, String clusterName) throws IOException { - Path certsDir = fileSystem.buildClusterSecurityDirPath(clusterName); - return uploadResource(resource, fileSystem, certsDir); - } - - /** - * Upload a local file to the cluster resources dir in HDFS. If the file - * already exists, it is not replaced. - * @param resource file to upload - * @param fileSystem file system - * @param roleName optional subdirectory (for component-specific resources) - * @param clusterName app name - * @return Path of the uploaded file - * @throws IOException file cannot be uploaded - */ - private Path uploadResource(File resource, SliderFileSystem fileSystem, - String roleName, String clusterName) throws IOException { - Path dir; - if (roleName == null) { - dir = fileSystem.buildClusterResourcePath(clusterName); - } else { - dir = fileSystem.buildClusterResourcePath(clusterName, roleName); - } - return uploadResource(resource, fileSystem, dir); - } - - /** - * Upload a local file to a specified HDFS directory. If the file already - * exists, it is not replaced. - * @param resource file to upload - * @param fileSystem file system - * @param parentDir destination directory in HDFS - * @return Path of the uploaded file - * @throws IOException file cannot be uploaded - */ - private synchronized Path uploadResource(File resource, - SliderFileSystem fileSystem, Path parentDir) throws IOException { - if (!fileSystem.getFileSystem().exists(parentDir)) { - fileSystem.getFileSystem().mkdirs(parentDir, + // 1. Create all config files for a component on hdfs for localization + // 2. Add the config file to localResource + //TODO handle Template format config file + public void createConfigFileAndAddLocalResource(ContainerLauncher launcher, + SliderFileSystem fs, String appName, Component component, + Map tokensForSubstitution, + StateAccessForProviders amState) throws IOException { + Path compDir = + fs.buildClusterResourcePath(appName, component.getName()); + if (!fs.getFileSystem().exists(compDir)) { + fs.getFileSystem().mkdirs(compDir, new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE)); - } - Path destPath = new Path(parentDir, resource.getName()); - if (!fileSystem.getFileSystem().exists(destPath)) { - FSDataOutputStream os = null; - try { - os = fileSystem.getFileSystem().create(destPath); - byte[] contents = FileUtils.readFileToByteArray(resource); - os.write(contents, 0, contents.length); - os.flush(); - } finally { - IOUtils.closeStream(os); - } - log.info("Uploaded {} to localization path {}", resource, destPath); + log.info("Creating component dir: " + compDir); } else { - log.info("Resource {} already existed at localization path {}", resource, - destPath); - } - - while (!fileSystem.getFileSystem().exists(destPath)) { - try { - Thread.sleep(500); - } catch (InterruptedException e) { - // ignore - } - } - - fileSystem.getFileSystem().setPermission(destPath, - new FsPermission(FsAction.READ, FsAction.NONE, FsAction.NONE)); - - return destPath; - } - - /** - * Write a configuration property map to a local file in a specified format. - * @param fileSystem file system - * @param file destination file - * @param configFormat file format - * @param configFileDN file description - * @param config properties to save to the file - * @param clusterName app name - * @throws IOException file cannot be created - */ - private synchronized void createConfigFile(SliderFileSystem fileSystem, - File file, ConfigFormat configFormat, String configFileDN, - Map config, String clusterName) throws IOException { - if (file.exists()) { - log.info("Skipping writing {} file {} because it already exists", - configFormat, file); - return; - } - log.info("Writing {} file {}", configFormat, file); - - ConfigUtils.prepConfigForTemplateOutputter(configFormat, config, - fileSystem, clusterName, file.getName()); - PublishedConfiguration publishedConfiguration = - new PublishedConfiguration(configFileDN, - config.entrySet()); - PublishedConfigurationOutputter configurationOutputter = - PublishedConfigurationOutputter.createOutputter(configFormat, - publishedConfiguration); - configurationOutputter.save(file); - } - - /** - * Determine config files requested in the appConf, generate the files, and - * localize them. - * @param launcher container launcher - * @param roleName component name - * @param roleGroup component group - * @param appConf app configurations - * @param configs configurations grouped by config name - * @param env environment variables - * @param fileSystem file system - * @param clusterName app name - * @throws IOException file(s) cannot be uploaded - * @throws BadConfigException file name not specified or file format not - * supported - */ - public void localizeConfigFiles(ContainerLauncher launcher, - String roleName, String roleGroup, - ConfTreeOperations appConf, - Map> configs, - MapOperations env, - SliderFileSystem fileSystem, - String clusterName) - throws IOException, BadConfigException { - for (Entry> configEntry : configs.entrySet()) { - String configFileName = appConf.getComponentOpt(roleGroup, - OptionKeys.CONF_FILE_PREFIX + configEntry.getKey() + OptionKeys - .NAME_SUFFIX, null); - String configFileType = appConf.getComponentOpt(roleGroup, - OptionKeys.CONF_FILE_PREFIX + configEntry.getKey() + OptionKeys - .TYPE_SUFFIX, null); - if (configFileName == null && configFileType == null) { - // config file not requested, so continue - continue; - } - if (configFileName == null) { - throw new BadConfigException("Config file name null for " + - configEntry.getKey()); - } - if (configFileType == null) { - throw new BadConfigException("Config file type null for " + - configEntry.getKey()); - } - ConfigFormat configFormat = ConfigFormat.resolve(configFileType); - if (configFormat == null) { - throw new BadConfigException("Config format " + configFileType + - " doesn't exist"); - } - boolean perComponent = appConf.getComponentOptBool(roleGroup, - OptionKeys.CONF_FILE_PREFIX + configEntry.getKey() + OptionKeys - .PER_COMPONENT, false); - boolean perGroup = appConf.getComponentOptBool(roleGroup, - OptionKeys.CONF_FILE_PREFIX + configEntry.getKey() + OptionKeys - .PER_GROUP, false); - - localizeConfigFile(launcher, roleName, roleGroup, configEntry.getKey(), - configFormat, configFileName, configs, env, fileSystem, - clusterName, perComponent, perGroup); - } - } - - /** - * Create and localize a config file. - * @param launcher container launcher - * @param roleName component name - * @param roleGroup component group - * @param configFileDN config description/name - * @param configFormat config format - * @param configFileName config file name - * @param configs configs grouped by config description/name - * @param env environment variables - * @param fileSystem file system - * @param clusterName app name - * @param perComponent true if file should be created per unique component - * @param perGroup true if file should be created per component group - * @throws IOException file cannot be uploaded - */ - public void localizeConfigFile(ContainerLauncher launcher, - String roleName, String roleGroup, - String configFileDN, ConfigFormat configFormat, String configFileName, - Map> configs, - MapOperations env, - SliderFileSystem fileSystem, - String clusterName, - boolean perComponent, - boolean perGroup) - throws IOException { - if (launcher == null) { + log.info("Component conf dir already exists: " + compDir); return; } - Map config = ConfigUtils.replacePropsInConfig( - configs.get(configFileDN), env.options); - String fileName = ConfigUtils.replaceProps(config, configFileName); - File localFile = new File(RESOURCE_DIR); - if (!localFile.exists()) { - if (!localFile.mkdir() && !localFile.exists()) { - throw new IOException(RESOURCE_DIR + " could not be created!"); - } - } - - String folder = null; - if (perComponent) { - folder = roleName; - } else if (perGroup) { - folder = roleGroup; - } - if (folder != null) { - localFile = new File(localFile, folder); - if (!localFile.exists()) { - if (!localFile.mkdir() && !localFile.exists()) { - throw new IOException(localFile + " could not be created!"); - } - } - } - localFile = new File(localFile, new File(fileName).getName()); - - log.info("Localizing {} configs to config file {} (destination {}) " + - "based on {} configs", config.size(), localFile, fileName, - configFileDN); - if (!localFile.exists()) { - createConfigFile(fileSystem, localFile, configFormat, configFileDN, - config, clusterName); - } else { - log.info("Local {} file {} already exists", configFormat, localFile); - } - Path destPath = uploadResource(localFile, fileSystem, folder, clusterName); - LocalResource configResource = fileSystem.createAmResource(destPath, - LocalResourceType.FILE); - - File destFile = new File(fileName); - if (destFile.isAbsolute()) { - launcher.addLocalResource( - RESOURCE_DIR + "/" + destFile.getName(), - configResource, fileName); - } else { - launcher.addLocalResource(APP_CONF_DIR + "/" + fileName, - configResource); - } - } - - /** - * Localize application tarballs and other resources requested by the app. - * @param launcher container launcher - * @param fileSystem file system - * @param appConf app configurations - * @param roleGroup component group - * @param clusterName app name - * @throws IOException resources cannot be uploaded - * @throws BadConfigException package name or type is not specified - */ - public void localizePackages(ContainerLauncher launcher, - SliderFileSystem fileSystem, ConfTreeOperations appConf, String roleGroup, - String clusterName) throws IOException, BadConfigException { - for (Entry> pkg : - getPackages(roleGroup, appConf).entrySet()) { - String pkgName = pkg.getValue().get(OptionKeys.NAME_SUFFIX); - String pkgType = pkg.getValue().get(OptionKeys.TYPE_SUFFIX); - Path pkgPath = fileSystem.buildResourcePath(pkgName); - if (!fileSystem.isFile(pkgPath)) { - pkgPath = fileSystem.buildResourcePath(clusterName, - pkgName); - } - if (!fileSystem.isFile(pkgPath)) { - throw new IOException("Package doesn't exist as a resource: " + - pkgName); - } - log.info("Adding resource {}", pkgName); - LocalResourceType type = LocalResourceType.FILE; - if ("archive".equals(pkgType)) { - type = LocalResourceType.ARCHIVE; - } - LocalResource packageResource = fileSystem.createAmResource( - pkgPath, type); - launcher.addLocalResource(APP_PACKAGES_DIR, packageResource); - } - } - - /** - * Build a map of configuration description/name to configuration key/value - * properties, with all known tokens substituted into the property values. - * @param appConf app configurations - * @param internalsConf internal configurations - * @param containerId container ID - * @param roleName component name - * @param roleGroup component group - * @param amState access to AM state - * @return configuration properties grouped by config description/name - */ - public Map> buildConfigurations( - ConfTreeOperations appConf, ConfTreeOperations internalsConf, - String containerId, String clusterName, String roleName, String roleGroup, - StateAccessForProviders amState) { - - Map> configurations = new TreeMap<>(); - Map tokens = getStandardTokenMap(appConf, - internalsConf, roleName, roleGroup, containerId, clusterName); - - Set configs = new HashSet<>(); - configs.addAll(getApplicationConfigurationTypes(roleGroup, appConf)); - configs.addAll(getSystemConfigurationsRequested(appConf)); - - for (String configType : configs) { - addNamedConfiguration(configType, appConf.getGlobalOptions().options, - configurations, tokens, amState); - if (appConf.getComponent(roleGroup) != null) { - addNamedConfiguration(configType, - appConf.getComponent(roleGroup).options, configurations, tokens, - amState); - } - } - - //do a final replacement of re-used configs - dereferenceAllConfigs(configurations); - - return configurations; - } - - /** - * Substitute "site." prefixed configuration values into other configuration - * values where needed. The format for these substitutions is that - * {@literal ${@//site/configDN/key}} will be replaced by the value for the - * "site.configDN.key" property. - * @param configurations configuration properties grouped by config - * description/name - */ - public void dereferenceAllConfigs( - Map> configurations) { - Map allConfigs = new HashMap<>(); - String lookupFormat = "${@//site/%s/%s}"; - for (Map.Entry> entry : configurations.entrySet()) { - Map configBucket = entry.getValue(); - for(Map.Entry config: configBucket.entrySet()) { - allConfigs.put(String.format(lookupFormat, entry.getKey(), config.getKey()), - config.getValue()); - } - } - - boolean finished = false; - while (!finished) { - finished = true; - for (Map.Entry entry : allConfigs.entrySet()) { - String configValue = entry.getValue(); - for (Map.Entry lookUpEntry : allConfigs.entrySet()) { - String lookUpValue = lookUpEntry.getValue(); - if (lookUpValue.contains("${@//site/")) { - continue; - } - String lookUpKey = lookUpEntry.getKey(); - if (configValue != null && configValue.contains(lookUpKey)) { - configValue = configValue.replace(lookUpKey, lookUpValue); - } - } - if (configValue != null && !configValue.equals(entry.getValue())) { - finished = false; - allConfigs.put(entry.getKey(), configValue); - } - } - } - for (Map.Entry> configEntry : configurations - .entrySet()) { - Map configBucket = configEntry.getValue(); - for (Map.Entry entry: configBucket.entrySet()) { - String configName = entry.getKey(); - String configValue = entry.getValue(); - for (Map.Entry lookUpEntry : allConfigs.entrySet()) { - String lookUpValue = lookUpEntry.getValue(); - if (lookUpValue.contains("${@//site/")) { - continue; - } - String lookUpKey = lookUpEntry.getKey(); - if (configValue != null && configValue.contains(lookUpKey)) { - configValue = configValue.replace(lookUpKey, lookUpValue); - } - } - configBucket.put(configName, configValue); - } - } - } - /** - * Return a set of configuration description/names represented in the app. - * configuration - * @param roleGroup component group - * @param appConf app configurations - * @return set of configuration description/names - */ - public Set getApplicationConfigurationTypes(String roleGroup, - ConfTreeOperations appConf) { - Set configList = new HashSet<>(); - - String prefix = OptionKeys.CONF_FILE_PREFIX; - String suffix = OptionKeys.TYPE_SUFFIX; - MapOperations component = appConf.getComponent(roleGroup); - if (component != null) { - addConfsToList(component, configList, prefix, suffix); - } - addConfsToList(appConf.getGlobalOptions(), configList, prefix, suffix); - - return configList; - } - - /** - * Finds all configuration description/names of the form - * prefixconfigDNsuffix in the configuration (e.g. conf.configDN.type). - * @param confMap configuration properties - * @param confList set containing configuration description/names - * @param prefix configuration key prefix to match - * @param suffix configuration key suffix to match - */ - private void addConfsToList(Map confMap, - Set confList, String prefix, String suffix) { - for (Entry entry : confMap.entrySet()) { - String key = entry.getKey(); - if (key.startsWith(prefix) && key.endsWith(suffix)) { - String confName = key.substring(prefix.length(), - key.length() - suffix.length()); - if (!confName.isEmpty()) { - confList.add(confName); - } + for (ConfigFile configFile : component.getConfiguration().getFiles()) { + String fileName = configFile.getSrcFile(); + // substitute file name + for (Map.Entry token : tokensForSubstitution.entrySet()) { + configFile.setDestFile(configFile.getDestFile() + .replaceAll(Pattern.quote(token.getKey()), token.getValue())); } - } - } - - /** - * Build a map of package description/name to package key/value properties - * (there should be two properties, type and name). - * @param roleGroup component group - * @param appConf app configurations - * @return map of package description/name to package key/value properties - * @throws BadConfigException package name or type is not specified - */ - public Map> getPackages(String roleGroup, - ConfTreeOperations appConf) throws BadConfigException { - Map> packages = new HashMap<>(); - String prefix = OptionKeys.PKG_FILE_PREFIX; - String typeSuffix = OptionKeys.TYPE_SUFFIX; - String nameSuffix = OptionKeys.NAME_SUFFIX; - MapOperations component = appConf.getComponent(roleGroup); - if (component == null) { - component = appConf.getGlobalOptions(); - } - for (Map.Entry entry : component.entrySet()) { - String key = entry.getKey(); - if (key.startsWith(prefix)) { - String confName; - String type; - if (key.endsWith(typeSuffix)) { - confName = key.substring(prefix.length(), key.length() - typeSuffix.length()); - type = typeSuffix; - } else if (key.endsWith(nameSuffix)) { - confName = key.substring(prefix.length(), key.length() - nameSuffix.length()); - type = nameSuffix; - } else { - continue; - } - if (!packages.containsKey(confName)) { - packages.put(confName, new HashMap()); + // substitute configs + substituteConfigs(configFile.getProps(), tokensForSubstitution); + + // write configs onto hdfs + PublishedConfiguration publishedConfiguration = + new PublishedConfiguration(fileName, + configFile.getProps().entrySet()); + Path remoteFile = new Path(compDir, fileName); + if (!fs.getFileSystem().exists(remoteFile)) { + PublishedConfigurationOutputter configurationOutputter = + PublishedConfigurationOutputter.createOutputter( + ConfigFormat.resolve(configFile.getType().toString()), + publishedConfiguration); + FSDataOutputStream os = null; + try { + os = fs.getFileSystem().create(remoteFile); + configurationOutputter.save(os); + os.flush(); + log.info("Created config file on hdfs: " + remoteFile); + } finally { + IOUtils.closeStream(os); } - packages.get(confName).put(type, entry.getValue()); - } - } - - for (Entry> pkg : packages.entrySet()) { - if (!pkg.getValue().containsKey(OptionKeys.TYPE_SUFFIX)) { - throw new BadConfigException("Package " + pkg.getKey() + " doesn't " + - "have a package type"); } - if (!pkg.getValue().containsKey(OptionKeys.NAME_SUFFIX)) { - throw new BadConfigException("Package " + pkg.getKey() + " doesn't " + - "have a package name"); - } - } - - return packages; - } - - /** - * Return system configurations requested by the app. - * @param appConf app configurations - * @return set of system configurations - */ - public Set getSystemConfigurationsRequested( - ConfTreeOperations appConf) { - Set configList = new HashSet<>(); - String configTypes = appConf.get(SYSTEM_CONFIGS); - if (configTypes != null && configTypes.length() > 0) { - String[] configs = configTypes.split(","); - for (String config : configs) { - configList.add(config.trim()); + // Publish configs + amState.getPublishedSliderConfigurations() + .put(configFile.getSrcFile(), publishedConfiguration); + + // Add resource for localization + LocalResource configResource = + fs.createAmResource(remoteFile, LocalResourceType.FILE); + File destFile = new File(configFile.getDestFile()); + //TODO why to we need to differetiate RESOURCE_DIR vs APP_CONF_DIR + if (destFile.isAbsolute()) { + String symlink = RESOURCE_DIR + "/" + fileName; + launcher.addLocalResource(symlink, configResource, + configFile.getDestFile()); + log.info("Add config file for localization: " + symlink + " -> " + + configResource.getResource().getFile() + ", dest mount path: " + + configFile.getDestFile()); + } else { + String symlink = APP_CONF_DIR + "/" + fileName; + launcher.addLocalResource(symlink, configResource); + log.info("Add config file for localization: " + symlink + " -> " + + configResource.getResource().getFile()); } } - - return configList; - } - - /** - * For a given config description/name, pull out its site configs from the - * source config map, remove the site.configDN. prefix from them, and place - * them into a new config map using the {@link #propagateSiteOptions} method - * (with tokens substituted). This new k/v map is put as the value for the - * configDN key in the configurations map. - * @param configName config description/name - * @param sourceConfig config containing site.* properties - * @param configurations configuration map to be populated - * @param tokens initial substitution tokens - * @param amState access to AM state - */ - private void addNamedConfiguration(String configName, - Map sourceConfig, - Map> configurations, - Map tokens, StateAccessForProviders amState) { - Map config = new HashMap<>(); - if (configName.equals(GLOBAL_CONFIG_TAG)) { - addDefaultGlobalConfig(config); - } - // add role hosts to tokens - addRoleRelatedTokens(tokens, amState); - propagateSiteOptions(sourceConfig, config, configName, tokens); - - configurations.put(configName, config); } /** * Get initial token map to be substituted into config values. * @param appConf app configurations - * @param internals internal configurations - * @param componentName component name - * @param componentGroup component group - * @param clusterName app name - * @return tokens to replace - */ - public Map getStandardTokenMap(ConfTreeOperations appConf, - ConfTreeOperations internals, String componentName, - String componentGroup, String clusterName) { - return getStandardTokenMap(appConf, internals, componentName, - componentGroup, null, clusterName); - } - - /** - * Get initial token map to be substituted into config values. - * @param appConf app configurations - * @param internals internal configurations * @param componentName component name * @param componentGroup component group * @param containerId container ID * @param clusterName app name * @return tokens to replace */ - public Map getStandardTokenMap(ConfTreeOperations appConf, - ConfTreeOperations internals, String componentName, + public Map getStandardTokenMap( + Configuration appConf, Configuration componentConf, String componentName, String componentGroup, String containerId, String clusterName) { Map tokens = new HashMap<>(); if (containerId != null) { tokens.put("${CONTAINER_ID}", containerId); } - String nnuri = appConf.get("site.fs.defaultFS"); - tokens.put("${NN_URI}", nnuri); - tokens.put("${NN_HOST}", URI.create(nnuri).getHost()); - tokens.put("${ZK_HOST}", appConf.get(OptionKeys.ZOOKEEPER_HOSTS)); - tokens.put("${DEFAULT_ZK_PATH}", appConf.get(OptionKeys.ZOOKEEPER_PATH)); - String prefix = appConf.getComponentOpt(componentGroup, ROLE_PREFIX, - null); + String nnuri = appConf.getProperty("fs.defaultFS"); + if (nnuri != null && !nnuri.isEmpty()) { + tokens.put("${NN_URI}", nnuri); + tokens.put("${NN_HOST}", URI.create(nnuri).getHost()); + } + tokens.put("${ZK_HOST}", appConf.getProperty(OptionKeys.ZOOKEEPER_HOSTS)); + tokens.put("${DEFAULT_ZK_PATH}", appConf.getProperty(OptionKeys.ZOOKEEPER_PATH)); + String prefix = componentConf.getProperty(ROLE_PREFIX); String dataDirSuffix = ""; if (prefix == null) { prefix = ""; } else { dataDirSuffix = "_" + SliderUtils.trimPrefix(prefix); } - tokens.put("${DEFAULT_DATA_DIR}", internals.getGlobalOptions() - .getOption(InternalKeys.INTERNAL_DATA_DIR_PATH, null) + dataDirSuffix); - tokens.put("${JAVA_HOME}", appConf.get(JAVA_HOME)); + tokens.put("${DEFAULT_DATA_DIR}", + appConf.getProperty(InternalKeys.INTERNAL_DATA_DIR_PATH) + + dataDirSuffix); + tokens.put("${JAVA_HOME}", appConf.getProperty(JAVA_HOME)); tokens.put("${COMPONENT_NAME}", componentName); tokens.put("${COMPONENT_NAME.lc}", componentName.toLowerCase()); tokens.put("${COMPONENT_PREFIX}", prefix); @@ -1005,7 +384,7 @@ private void addNamedConfiguration(String configName, * @param tokens existing tokens * @param amState access to AM state */ - public void addRoleRelatedTokens(Map tokens, + public void addRoleHostTokens(Map tokens, StateAccessForProviders amState) { if (amState == null) { return; @@ -1020,26 +399,6 @@ public void addRoleRelatedTokens(Map tokens, } /** - * Add global configuration properties. - * @param config map where default global properties will be added - */ - private void addDefaultGlobalConfig(Map config) { - config.put("app_log_dir", "${LOG_DIR}"); - config.put("app_pid_dir", "${WORK_DIR}/app/run"); - config.put("app_install_dir", "${WORK_DIR}/app/install"); - config.put("app_conf_dir", "${WORK_DIR}/" + APP_CONF_DIR); - config.put("app_input_conf_dir", "${WORK_DIR}/" + PROPAGATED_CONF_DIR_NAME); - - // add optional parameters only if they are not already provided - if (!config.containsKey("pid_file")) { - config.put("pid_file", "${WORK_DIR}/app/run/component.pid"); - } - if (!config.containsKey("app_root")) { - config.put("app_root", "${WORK_DIR}/app/install"); - } - } - - /** * Return a list of hosts based on current ClusterNodes. * @param values cluster nodes * @param hostOnly whether host or host/server name will be added to list @@ -1101,82 +460,4 @@ public void updateServiceRecord(StateAccessForProviders amState, containerId, e); } } - - /** - * Publish a named property bag that may contain name-value pairs for app - * configurations such as hbase-site. - * @param name config file identifying name - * @param description config file description - * @param entries config file properties - * @param amState access to AM state - */ - public void publishApplicationInstanceData(String name, String description, - Iterable> entries, - StateAccessForProviders amState) { - PublishedConfiguration pubconf = new PublishedConfiguration(description, - entries); - log.info("publishing {}", pubconf); - amState.getPublishedSliderConfigurations().put(name, pubconf); - } - - /** - * Publish an export group. - * @param exportGroup export groups - * @param amState access to AM state - * @param groupName export group name - */ - public void publishExportGroup( - Map> exportGroup, - StateAccessForProviders amState, String groupName) { - // Publish in old format for the time being - Map simpleEntries = new HashMap<>(); - for (Entry> entry : exportGroup.entrySet()) { - Set exports = entry.getValue(); - if (SliderUtils.isNotEmpty(exports)) { - Set values = new TreeSet<>(); - for (ExportEntry export : exports) { - values.add(export.getValue()); - } - simpleEntries.put(entry.getKey(), StringUtils.join(",", values)); - } - } - publishApplicationInstanceData(groupName, groupName, - simpleEntries.entrySet(), amState); - - PublishedExports exports = new PublishedExports(groupName); - exports.setUpdated(new Date().getTime()); - exports.putValues(exportGroup.entrySet()); - amState.getPublishedExportsSet().put(groupName, exports); - } - - public Map getExports(ConfTreeOperations appConf, - String roleGroup) { - Map exports = new HashMap<>(); - propagateOptions(appConf.getComponent(roleGroup).options, exports, - null, OptionKeys.EXPORT_PREFIX); - return exports; - } - - public String getGroupKey(String roleGroup, ConfTreeOperations appConf) { - String rolePrefix = appConf.getComponentOpt(roleGroup, ROLE_PREFIX, ""); - return getNameOrGroupKey(rolePrefix, roleGroup); - } - - public String getNameKey(String roleName, String roleGroup, - ConfTreeOperations appConf) { - String rolePrefix = appConf.getComponentOpt(roleGroup, ROLE_PREFIX, ""); - return getNameOrGroupKey(rolePrefix, roleName); - } - - public String getNameOrGroupKey(String rolePrefix, String roleNameOrGroup) { - if (!rolePrefix.isEmpty()) { - if (!roleNameOrGroup.startsWith(rolePrefix)) { - log.warn("Something went wrong, {} doesn't start with {}", - roleNameOrGroup, rolePrefix); - return null; - } - roleNameOrGroup = roleNameOrGroup.substring(rolePrefix.length()); - } - return roleNameOrGroup.toUpperCase(Locale.ENGLISH); - } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java index 86d87ac..8b88c28 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java @@ -22,7 +22,6 @@ import org.apache.slider.common.tools.SliderFileSystem; import org.apache.slider.core.conf.AggregateConf; import org.apache.slider.core.conf.ConfTreeOperations; -import org.apache.slider.core.exceptions.BadConfigException; import org.apache.slider.core.exceptions.SliderException; import org.apache.slider.providers.AbstractClientProvider; import org.apache.slider.providers.ProviderRole; @@ -30,14 +29,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.Set; -import static org.apache.slider.providers.docker.DockerKeys.DOCKER_IMAGE; - public class DockerClientProvider extends AbstractClientProvider implements SliderKeys { @@ -64,35 +59,7 @@ public String getName() { public void validateInstanceDefinition(AggregateConf instanceDefinition, SliderFileSystem fs) throws SliderException { super.validateInstanceDefinition(instanceDefinition, fs); - - ConfTreeOperations appConf = instanceDefinition.getAppConfOperations(); - ConfTreeOperations resources = instanceDefinition.getResourceOperations(); - - for (String roleGroup : resources.getComponentNames()) { - if (roleGroup.equals(COMPONENT_AM)) { - continue; - } - if (appConf.getComponentOpt(roleGroup, DOCKER_IMAGE, null) == null && - appConf.getGlobalOptions().get(DOCKER_IMAGE) == null) { - throw new BadConfigException("Property " + DOCKER_IMAGE + " not " + - "specified for " + roleGroup); - } - - providerUtils.getPackages(roleGroup, appConf); - - if (appConf.getComponentOptBool(roleGroup, AM_CONFIG_GENERATION, false)) { - // build and localize configuration files - Map> configurations = - providerUtils.buildConfigurations(appConf, appConf, null, - null, roleGroup, roleGroup, null); - try { - providerUtils.localizeConfigFiles(null, roleGroup, roleGroup, appConf, - configurations, null, fs, null); - } catch (IOException e) { - throw new BadConfigException(e.toString()); - } - } - } + //TODO validate Application payload, part of that is already done in ApplicationApiService, need to do more } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java index 63416cc..a39b2ee 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java @@ -17,29 +17,22 @@ */ package org.apache.slider.providers.docker; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.registry.client.types.ServiceRecord; import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerStatus; -import org.apache.hadoop.yarn.api.records.LocalResource; -import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.slider.api.ClusterDescription; import org.apache.slider.api.ClusterNode; -import org.apache.slider.api.OptionKeys; +import org.apache.slider.api.resource.Application; +import org.apache.slider.api.resource.Component; import org.apache.slider.common.SliderKeys; import org.apache.slider.common.tools.SliderFileSystem; import org.apache.slider.common.tools.SliderUtils; -import org.apache.slider.core.conf.AggregateConf; -import org.apache.slider.core.conf.ConfTreeOperations; -import org.apache.slider.core.conf.MapOperations; import org.apache.slider.core.exceptions.SliderException; import org.apache.slider.core.launch.CommandLineBuilder; import org.apache.slider.core.launch.ContainerLauncher; -import org.apache.slider.core.registry.docstore.ConfigFormat; -import org.apache.slider.core.registry.docstore.ConfigUtils; import org.apache.slider.core.registry.docstore.ExportEntry; +import org.apache.slider.core.registry.docstore.PublishedConfiguration; import org.apache.slider.providers.AbstractProviderService; import org.apache.slider.providers.MonitorDetail; import org.apache.slider.providers.ProviderCore; @@ -49,38 +42,27 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; import java.io.IOException; -import java.net.URL; import java.util.Collections; -import java.util.Date; -import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Map.Entry; -import java.util.Scanner; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import java.util.regex.Pattern; -public class DockerProviderService extends AbstractProviderService implements - ProviderCore, - DockerKeys, - SliderKeys { +public class DockerProviderService extends AbstractProviderService + implements ProviderCore, DockerKeys, SliderKeys { protected static final Logger log = LoggerFactory.getLogger(DockerProviderService.class); private static final ProviderUtils providerUtils = new ProviderUtils(log); - private static final String EXPORT_GROUP = "quicklinks"; + private static final String QUICK_LINKS = "quicklinks"; private static final String APPLICATION_TAG = "application"; private static final String HOST_KEY_FORMAT = "${%s_HOST}"; private static final String IP_KEY_FORMAT = "${%s_IP}"; private static final String VARIABLE_INDICATOR = "${"; - private String clusterName = null; - private SliderFileSystem fileSystem = null; - private final Map> exportMap = new ConcurrentHashMap<>(); @@ -99,184 +81,74 @@ public boolean isSupportedRole(String role) { } @Override - public void validateInstanceDefinition(AggregateConf instanceDefinition) - throws SliderException { - } - - private String getClusterName() { - if (SliderUtils.isUnset(clusterName)) { - clusterName = getAmState().getInternalsSnapshot().get(OptionKeys.APPLICATION_NAME); - } - return clusterName; - } - - @Override public void buildContainerLaunchContext(ContainerLauncher launcher, - AggregateConf instanceDefinition, Container container, - ProviderRole providerRole, SliderFileSystem fileSystem, - Path generatedConfPath, MapOperations resourceComponent, - MapOperations appComponent, Path containerTmpDirPath) + Application application, Container container, ProviderRole providerRole, + SliderFileSystem fileSystem) throws IOException, SliderException { String roleName = providerRole.name; String roleGroup = providerRole.group; - log.info("Build launch context for Docker"); - log.debug(instanceDefinition.toString()); - - ConfTreeOperations appConf = instanceDefinition.getAppConfOperations(); + Component component = providerRole.component; launcher.setYarnDockerMode(true); - launcher.setDockerImage(appConf.getComponentOpt(roleGroup, DOCKER_IMAGE, - null)); - launcher.setDockerNetwork(appConf.getComponentOpt(roleGroup, DOCKER_NETWORK, - DEFAULT_DOCKER_NETWORK)); - launcher.setRunPrivilegedContainer(appConf.getComponentOptBool(roleGroup, - DOCKER_USE_PRIVILEGED, DEFAULT_DOCKER_USE_PRIVILEGED)); - - // Set the environment - Map standardTokens = providerUtils.getStandardTokenMap( - getAmState().getAppConfSnapshot(), getAmState().getInternalsSnapshot(), - roleName, roleGroup, container.getId().toString(), getClusterName()); - Map replaceTokens = providerUtils.filterSiteOptions( - appConf.getComponent(roleGroup).options, standardTokens); - replaceTokens.putAll(standardTokens); - launcher.putEnv(SliderUtils.buildEnvMap(appComponent, replaceTokens)); - - String workDir = ApplicationConstants.Environment.PWD.$(); - launcher.setEnv("WORK_DIR", workDir); - log.info("WORK_DIR set to {}", workDir); - String logDir = ApplicationConstants.LOG_DIR_EXPANSION_VAR; - launcher.setEnv("LOG_DIR", logDir); - log.info("LOG_DIR set to {}", logDir); + launcher.setDockerImage(component.getArtifact().getId()); + launcher.setDockerNetwork(component.getConfiguration() + .getProperty(DOCKER_NETWORK, DEFAULT_DOCKER_NETWORK)); + launcher.setRunPrivilegedContainer(component.getRunPrivilegedContainer()); + + // Generate tokens (key-value pair) for config substitution. + Map standardTokens = providerUtils + .getStandardTokenMap(application.getConfiguration(), + component.getConfiguration(), roleName, roleGroup, + container.getId().toString(), application.getName()); + Map tokensForSubstitution = providerUtils.substituteConfigs( + component.getConfiguration().getProperties(), standardTokens); + + tokensForSubstitution.putAll(standardTokens); + + // Set the environment variables + launcher.putEnv(SliderUtils + .buildEnvMap(component.getConfiguration(), tokensForSubstitution)); + launcher.setEnv("WORK_DIR", ApplicationConstants.Environment.PWD.$()); + launcher.setEnv("LOG_DIR", ApplicationConstants.LOG_DIR_EXPANSION_VAR); if (System.getenv(HADOOP_USER_NAME) != null) { launcher.setEnv(HADOOP_USER_NAME, System.getenv(HADOOP_USER_NAME)); } - //add english env launcher.setEnv("LANG", "en_US.UTF-8"); launcher.setEnv("LC_ALL", "en_US.UTF-8"); launcher.setEnv("LANGUAGE", "en_US.UTF-8"); - //local resources - providerUtils.localizePackages(launcher, fileSystem, appConf, roleGroup, - getClusterName()); - - if (SliderUtils.isHadoopClusterSecure(getConfig())) { - providerUtils.localizeServiceKeytabs(launcher, instanceDefinition, - fileSystem, getClusterName()); + for (Entry entry : launcher.getEnv().entrySet()) { + tokensForSubstitution.put("${" + entry.getKey() + "}", entry.getValue()); } - if (appComponent.getOptionBool(AM_CONFIG_GENERATION, false)) { - // build and localize configuration files - Map> configurations = - providerUtils.buildConfigurations( - instanceDefinition.getAppConfOperations(), - instanceDefinition.getInternalOperations(), - container.getId().toString(), getClusterName(), - roleName, roleGroup, getAmState()); - providerUtils.localizeConfigFiles(launcher, roleName, roleGroup, - appConf, configurations, launcher.getEnv(), fileSystem, - getClusterName()); + providerUtils.addRoleHostTokens(tokensForSubstitution, amState); + + log.info("Token for substitution: " + tokensForSubstitution); + + if (SliderUtils.isHadoopClusterSecure(getConfig())) { + //TODO localize key tabs, WHY is this code needed ? WHY DOES CONTAINER REQUIRE AM KEYTAB?? + providerUtils.localizeServiceKeytabs(launcher, fileSystem, application); } - //add the configuration resources - launcher.addLocalResources(fileSystem.submitDirectory( - generatedConfPath, - PROPAGATED_CONF_DIR_NAME)); + // create config file on hdfs and add local resource + providerUtils.createConfigFileAndAddLocalResource(launcher, fileSystem, + application.getName(), component, tokensForSubstitution, getAmState()); CommandLineBuilder operation = new CommandLineBuilder(); - operation.add(appConf.getComponentOpt(roleGroup, DOCKER_START_COMMAND, - "/bin/bash")); - + operation.add(component.getLaunchCommand()); operation.add("> " + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" + OUT_FILE + " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" + ERR_FILE); - launcher.addCommand(operation.build()); - // Additional files to localize - String appResourcesString = instanceDefinition.getAppConfOperations() - .getGlobalOptions().getOption(APP_RESOURCES, null); - log.info("Configuration value for extra resources to localize: {}", appResourcesString); - if (null != appResourcesString) { - try (Scanner scanner = new Scanner(appResourcesString).useDelimiter(",")) { - while (scanner.hasNext()) { - String resource = scanner.next(); - Path resourcePath = new Path(resource); - LocalResource extraResource = fileSystem.createAmResource( - fileSystem.getFileSystem().resolvePath(resourcePath), - LocalResourceType.FILE); - String destination = APP_RESOURCES_DIR + "/" + resourcePath.getName(); - log.info("Localizing {} to {}", resourcePath, destination); - // TODO Can we try harder to avoid collisions? - launcher.addLocalResource(destination, extraResource); - } - } - } - } - - @Override - public void initializeApplicationConfiguration( - AggregateConf instanceDefinition, SliderFileSystem fileSystem, - String roleGroup) - throws IOException, SliderException { - this.fileSystem = fileSystem; - } - - @Override - public void applyInitialRegistryDefinitions(URL amWebURI, - ServiceRecord serviceRecord) - throws IOException { - super.applyInitialRegistryDefinitions(amWebURI, serviceRecord); - - // identify client component - String clientName = null; - ConfTreeOperations appConf = getAmState().getAppConfSnapshot(); - for (String component : appConf.getComponentNames()) { - if (COMPONENT_TYPE_CLIENT.equals(appConf.getComponentOpt(component, - COMPONENT_TYPE_KEY, null))) { - clientName = component; - break; - } - } - if (clientName == null) { - log.info("No client component specified, not publishing client configs"); - return; - } - - // register AM-generated client configs - // appConf should already be resolved! - MapOperations clientOperations = appConf.getComponent(clientName); - if (!clientOperations.getOptionBool(AM_CONFIG_GENERATION, false)) { - log.info("AM config generation is false, not publishing client configs"); - return; - } - - // build and localize configuration files - Map> configurations = - providerUtils.buildConfigurations(appConf, getAmState() - .getInternalsSnapshot(), null, getClusterName(), clientName, - clientName, getAmState()); - - for (Map.Entry> entry : configurations.entrySet()) { - String configFileDN = entry.getKey(); - String configFileName = appConf.getComponentOpt(clientName, - OptionKeys.CONF_FILE_PREFIX + configFileDN + OptionKeys - .NAME_SUFFIX, null); - String configFileType = appConf.getComponentOpt(clientName, - OptionKeys.CONF_FILE_PREFIX + configFileDN + OptionKeys - .TYPE_SUFFIX, null); - if (configFileName == null || configFileType == null) { - continue; - } - ConfigFormat configFormat = ConfigFormat.resolve(configFileType); - - Map config = entry.getValue(); - ConfigUtils.prepConfigForTemplateOutputter(configFormat, config, - fileSystem, getClusterName(), - new File(configFileName).getName()); - providerUtils.publishApplicationInstanceData(configFileDN, configFileDN, - config.entrySet(), getAmState()); - } + // publish exports + // TODO move this to app level, no need to do this for every container launch + providerUtils + .substituteConfigs(application.getQuicklinks(), tokensForSubstitution); + PublishedConfiguration pubconf = new PublishedConfiguration(QUICK_LINKS, + application.getQuicklinks().entrySet()); + amState.getPublishedSliderConfigurations().put(QUICK_LINKS, pubconf); } @Override @@ -310,125 +182,13 @@ public boolean processContainerStatus(ContainerId containerId, return false; } - String roleName = instance.role; - String roleGroup = instance.group; - String containerIdStr = containerId.toString(); - providerUtils.updateServiceRecord(getAmState(), yarnRegistry, - containerIdStr, roleName, status.getIPs(), status.getHost()); + containerId.toString(), instance.role, status.getIPs(), status.getHost()); - publishExportGroups(containerIdStr, roleName, roleGroup, - status.getHost(), status.getIPs()); + // TODO publish ip and host return false; } - - /** - * This method looks for configuration properties of the form - * export.key,value and publishes the key,value pair. Standard tokens are - * substituted into the value, and COMPONENTNAME_HOST and THIS_HOST tokens - * are substituted with the actual hostnames of the containers. - */ - protected void publishExportGroups(String containerId, - String roleName, String roleGroup, String thisHost, List ips) { - ConfTreeOperations appConf = getAmState().getAppConfSnapshot(); - ConfTreeOperations internalsConf = getAmState().getInternalsSnapshot(); - - Map exports = providerUtils.getExports( - getAmState().getAppConfSnapshot(), roleGroup); - - // publish export groups if any - Map standardTokens = providerUtils.getStandardTokenMap( - appConf, internalsConf, roleName, roleGroup, containerId, - getClusterName()); - Map replaceTokens = providerUtils.filterSiteOptions( - appConf.getComponent(roleGroup).options, standardTokens); - replaceTokens.putAll(standardTokens); - - String roleNameKey = providerUtils.getNameKey(roleName, roleGroup, - appConf); - String roleNameIPKey = null; - if (roleNameKey != null) { - replaceTokens.put(String.format(HOST_KEY_FORMAT, roleNameKey), thisHost); - roleNameIPKey = Pattern.quote(String.format(IP_KEY_FORMAT, roleNameKey)); - } else { - // should not happen, but log if it does - log.info("Not replacing HOST or IP tokens because key was null for {}", - roleName); - } - String roleGroupKey = providerUtils.getGroupKey(roleGroup, appConf); - String roleGroupIPKey = null; - if (roleGroupKey != null) { - if (roleNameKey == null || !roleGroupKey.equals(roleNameKey)) { - replaceTokens.put(String.format(HOST_KEY_FORMAT, roleGroupKey), - thisHost); - roleGroupIPKey = Pattern.quote(String.format(IP_KEY_FORMAT, - roleGroupKey)); - } - } else { - // should not happen, but log if it does - log.info("Not replacing HOST or IP tokens because key was null for {}", - roleGroup); - } - replaceTokens.put("${THIS_HOST}", thisHost); - - for (Entry export : exports.entrySet()) { - String value = export.getValue(); - // replace host names and site properties - for (Map.Entry entry : replaceTokens.entrySet()) { - String token = entry.getKey(); - if (value.contains(token)) { - value = value.replaceAll(Pattern.quote(token), entry.getValue()); - } - } - Set values = new HashSet<>(); - for (String ip : ips) { - values.add(substituteIP(roleNameIPKey, roleGroupIPKey, ip, value)); - } - for (String exportValue : values) { - if (exportValue.contains(VARIABLE_INDICATOR)) { - // not all variables have been substituted, so do not export - continue; - } - ExportEntry entry = new ExportEntry(); - entry.setContainerId(containerId); - entry.setLevel(APPLICATION_TAG); - entry.setValue(exportValue); - entry.setUpdatedTime(new Date().toString()); - Set exportEntries = getExportEntries(export.getKey()); - exportEntries.add(entry); - log.info("Preparing to publish for {}. Key {} and Value {}", - roleName, export.getKey(), entry); - } - } - if (!exportMap.isEmpty()) { - providerUtils.publishExportGroup(exportMap, getAmState(), EXPORT_GROUP); - } - } - - protected String substituteIP(String roleNameIPKey, String roleGroupIPKey, - String ip, String value) { - if (roleNameIPKey != null) { - value = value.replaceAll(roleNameIPKey, ip); - } - if (roleGroupIPKey != null) { - value = value.replaceAll(roleGroupIPKey, ip); - } - return value; - } - - protected Set getExportEntries(String key) { - if (!this.exportMap.containsKey(key)) { - synchronized (this.exportMap) { - if (!this.exportMap.containsKey(key)) { - this.exportMap.put(key, Collections.newSetFromMap( - new ConcurrentHashMap<>())); - } - } - } - - return this.exportMap.get(key); - } - + @Override public Map buildMonitorDetails(ClusterDescription clusterDesc) { Map details = super.buildMonitorDetails(clusterDesc); @@ -436,6 +196,7 @@ protected String substituteIP(String roleNameIPKey, String roleGroupIPKey, return details; } + // build role -> list of hosts for the role private void buildRoleHostDetails(Map details) { for (Map.Entry> entry : getAmState().getRoleClusterNodeMapping().entrySet()) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java deleted file mode 100644 index b58d3aa..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java +++ /dev/null @@ -1,305 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.providers.slideram; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.yarn.api.records.LocalResource; -import org.apache.hadoop.yarn.api.records.LocalResourceType; -import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.slider.api.InternalKeys; -import org.apache.slider.api.ResourceKeys; -import org.apache.slider.api.RoleKeys; -import org.apache.slider.common.SliderKeys; -import org.apache.slider.common.SliderXmlConfKeys; -import org.apache.slider.common.tools.SliderFileSystem; -import org.apache.slider.common.tools.SliderUtils; -import org.apache.slider.core.conf.AggregateConf; -import org.apache.slider.core.conf.MapOperations; -import org.apache.slider.core.exceptions.BadClusterStateException; -import org.apache.slider.core.exceptions.BadConfigException; -import org.apache.slider.core.exceptions.SliderException; -import org.apache.slider.core.launch.AbstractLauncher; -import org.apache.slider.core.launch.JavaCommandLineBuilder; -import org.apache.slider.providers.AbstractClientProvider; -import org.apache.slider.providers.PlacementPolicy; -import org.apache.slider.providers.ProviderRole; -import org.apache.slider.providers.ProviderUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.apache.slider.api.ResourceKeys.COMPONENT_INSTANCES; - -/** - * handles the setup of the Slider AM. - * This keeps aspects of role, cluster validation and Clusterspec setup - * out of the core slider client - */ -public class SliderAMClientProvider extends AbstractClientProvider - implements SliderKeys { - - - protected static final Logger log = - LoggerFactory.getLogger(SliderAMClientProvider.class); - protected static final String NAME = "SliderAM"; - public static final String INSTANCE_RESOURCE_BASE = PROVIDER_RESOURCE_BASE_ROOT + - "slideram/instance/"; - public static final String INTERNAL_JSON = - INSTANCE_RESOURCE_BASE + "internal.json"; - public static final String APPCONF_JSON = - INSTANCE_RESOURCE_BASE + "appconf.json"; - public static final String RESOURCES_JSON = - INSTANCE_RESOURCE_BASE + "resources.json"; - - public SliderAMClientProvider(Configuration conf) { - super(conf); - } - - /** - * List of roles - */ - public static final List ROLES = - new ArrayList(); - - public static final int KEY_AM = ROLE_AM_PRIORITY_INDEX; - - public static final ProviderRole APPMASTER = - new ProviderRole(COMPONENT_AM, KEY_AM, - PlacementPolicy.EXCLUDE_FROM_FLEXING, - ResourceKeys.DEFAULT_NODE_FAILURE_THRESHOLD, - 0, ""); - - /** - * Initialize role list - */ - static { - ROLES.add(APPMASTER); - } - - @Override - public String getName() { - return NAME; - } - - @Override - public List getRoles() { - return ROLES; - } - - - @Override //Client - public void preflightValidateClusterConfiguration(SliderFileSystem sliderFileSystem, - String clustername, - Configuration configuration, - AggregateConf instanceDefinition, - Path clusterDirPath, - Path generatedConfDirPath, - boolean secure) - throws SliderException, IOException { - - super.preflightValidateClusterConfiguration(sliderFileSystem, clustername, configuration, instanceDefinition, clusterDirPath, generatedConfDirPath, secure); - //add a check for the directory being writeable by the current user - String - dataPath = instanceDefinition.getInternalOperations() - .getGlobalOptions() - .getMandatoryOption( - InternalKeys.INTERNAL_DATA_DIR_PATH); - - Path path = new Path(dataPath); - sliderFileSystem.verifyDirectoryWriteAccess(path); - Path historyPath = new Path(clusterDirPath, SliderKeys.HISTORY_DIR_NAME); - sliderFileSystem.verifyDirectoryWriteAccess(historyPath); - } - - /** - * Verify that an instance definition is considered valid by the provider - * @param instanceDefinition instance definition - * @throws SliderException if the configuration is not valid - */ - public void validateInstanceDefinition(AggregateConf instanceDefinition, SliderFileSystem fs) throws - SliderException { - - super.validateInstanceDefinition(instanceDefinition, fs); - - // make sure there is no negative entry in the instance count - Map> instanceMap = - instanceDefinition.getResources().components; - for (Map.Entry> entry : instanceMap.entrySet()) { - MapOperations mapOperations = new MapOperations(entry); - int instances = mapOperations.getOptionInt(COMPONENT_INSTANCES, 0); - if (instances < 0) { - throw new BadClusterStateException( - "Component %s has negative instance count: %d", - mapOperations.name, - instances); - } - } - } - - /** - * The Slider AM sets up all the dependency JARs above slider.jar itself - * {@inheritDoc} - */ - public void prepareAMAndConfigForLaunch(SliderFileSystem fileSystem, - Configuration serviceConf, - AbstractLauncher launcher, - AggregateConf instanceDescription, - Path snapshotConfDirPath, - Path generatedConfDirPath, - Configuration clientConfExtras, - String libdir, - Path tempPath, boolean miniClusterTestRun) - throws IOException, SliderException { - - Map providerResources = new HashMap<>(); - - ProviderUtils.addProviderJar(providerResources, - this, - SLIDER_JAR, - fileSystem, - tempPath, - libdir, - miniClusterTestRun); - - log.info("Loading all dependencies for AM."); - // If slider.tar.gz is available in hdfs use it, else upload all jars - Path dependencyLibTarGzip = fileSystem.getDependencyTarGzip(); - if (fileSystem.isFile(dependencyLibTarGzip)) { - SliderUtils.putAmTarGzipAndUpdate(providerResources, fileSystem); - } else { - for (String libDirProp : SliderUtils.getLibDirs()) { - ProviderUtils.addAllDependencyJars(providerResources, - fileSystem, - tempPath, - libdir, - libDirProp); - - } - } - addKeytabResourceIfNecessary(fileSystem, - instanceDescription, - providerResources); - - launcher.addLocalResources(providerResources); - - //also pick up all env variables from a map - launcher.copyEnvVars( - instanceDescription.getInternalOperations().getOrAddComponent( - SliderKeys.COMPONENT_AM)); - } - - /** - * If the cluster is secure, and an HDFS installed keytab is available for AM - * authentication, add this keytab as a local resource for the AM launch. - * - * @param fileSystem - * @param instanceDescription - * @param providerResources - * @throws IOException - * @throws BadConfigException if there's no keytab and it is explicitly required. - */ - protected void addKeytabResourceIfNecessary(SliderFileSystem fileSystem, - AggregateConf instanceDescription, - Map providerResources) - throws IOException, BadConfigException { - if (UserGroupInformation.isSecurityEnabled()) { - String keytabPathOnHost = instanceDescription.getAppConfOperations() - .getComponent(SliderKeys.COMPONENT_AM).get( - SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH); - if (SliderUtils.isUnset(keytabPathOnHost)) { - String amKeytabName = instanceDescription.getAppConfOperations() - .getComponent(SliderKeys.COMPONENT_AM).get( - SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME); - String keytabDir = instanceDescription.getAppConfOperations() - .getComponent(SliderKeys.COMPONENT_AM).get( - SliderXmlConfKeys.KEY_HDFS_KEYTAB_DIR); - Path keytabPath = fileSystem.buildKeytabPath(keytabDir, amKeytabName, - instanceDescription.getName()); - if (fileSystem.getFileSystem().exists(keytabPath)) { - LocalResource keytabRes = fileSystem.createAmResource(keytabPath, - LocalResourceType.FILE); - - providerResources.put(SliderKeys.KEYTAB_DIR + "/" + - amKeytabName, keytabRes); - } else { - log.warn("No keytab file was found at {}.", keytabPath); - if (getConf().getBoolean(KEY_AM_LOGIN_KEYTAB_REQUIRED, false)) { - throw new BadConfigException("No keytab file was found at %s.", keytabPath); - - } else { - log.warn("The AM will be " - + "started without a kerberos authenticated identity. " - + "The application is therefore not guaranteed to remain " - + "operational beyond 24 hours."); - } - } - } - } - } - - /** - * Update the AM resource with any local needs - * @param capability capability to update - */ - public void prepareAMResourceRequirements(MapOperations sliderAM, - Resource capability) { - capability.setMemory(sliderAM.getOptionInt( - ResourceKeys.YARN_MEMORY, - capability.getMemory())); - capability.setVirtualCores( - sliderAM.getOptionInt(ResourceKeys.YARN_CORES, capability.getVirtualCores())); - } - - /** - * Extract any JVM options from the cluster specification and - * add them to the command line - */ - public void addJVMOptions(AggregateConf aggregateConf, - JavaCommandLineBuilder cmdLine) - throws BadConfigException { - - MapOperations sliderAM = - aggregateConf.getAppConfOperations().getMandatoryComponent( - SliderKeys.COMPONENT_AM); - cmdLine.forceIPv4().headless(); - String heap = sliderAM.getOption(RoleKeys.JVM_HEAP, - DEFAULT_JVM_HEAP); - cmdLine.setJVMHeap(heap); - String jvmopts = sliderAM.getOption(RoleKeys.JVM_OPTS, ""); - if (SliderUtils.isSet(jvmopts)) { - cmdLine.add(jvmopts); - } - } - - - @Override - public void prepareInstanceConfiguration(AggregateConf aggregateConf) - throws SliderException, IOException { - mergeTemplates(aggregateConf, - INTERNAL_JSON, RESOURCES_JSON, APPCONF_JSON - ); - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java index c021b80..e0de366 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java @@ -18,20 +18,16 @@ package org.apache.slider.providers.slideram; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.registry.client.binding.RegistryTypeUtils; import org.apache.hadoop.registry.client.types.ServiceRecord; +import org.apache.slider.api.resource.Application; import org.apache.slider.common.SliderKeys; import org.apache.slider.common.tools.ConfigHelper; import org.apache.slider.common.tools.SliderFileSystem; import org.apache.slider.common.tools.SliderUtils; -import org.apache.slider.core.conf.AggregateConf; -import org.apache.slider.core.conf.MapOperations; -import org.apache.slider.core.exceptions.BadCommandArgumentsException; import org.apache.slider.core.exceptions.SliderException; import org.apache.slider.core.launch.ContainerLauncher; import org.apache.slider.core.registry.docstore.PublishedConfiguration; @@ -43,7 +39,6 @@ import org.apache.slider.server.appmaster.PublishedArtifacts; import org.apache.slider.server.appmaster.web.rest.RestPaths; -import java.io.File; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; @@ -70,24 +65,11 @@ public SliderAMProviderService() { public String getHumanName() { return "Slider Application"; } - - @Override - public Configuration loadProviderConfigurationInformation(File confDir) throws - BadCommandArgumentsException, - IOException { - return null; - } @Override public void buildContainerLaunchContext(ContainerLauncher containerLauncher, - AggregateConf instanceDefinition, - Container container, - ProviderRole role, - SliderFileSystem sliderFileSystem, - Path generatedConfPath, - MapOperations resourceComponent, - MapOperations appComponent, - Path containerTmpDirPath) throws IOException, SliderException { + Application application, Container container, ProviderRole role, + SliderFileSystem sliderFileSystem) throws IOException, SliderException { } @Override @@ -95,11 +77,6 @@ public void buildContainerLaunchContext(ContainerLauncher containerLauncher, return new ArrayList<>(0); } - @Override - public void validateInstanceDefinition(AggregateConf instanceDefinition) throws - SliderException { - - } @Override public void applyInitialRegistryDefinitions(URL amWebURI, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java index 3cfe167..70eab71 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java @@ -18,16 +18,12 @@ package org.apache.slider.server.appmaster; -import com.google.common.base.Preconditions; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.slider.common.SliderKeys; +import org.apache.slider.api.resource.Application; import org.apache.slider.common.tools.SliderFileSystem; -import org.apache.slider.core.conf.AggregateConf; -import org.apache.slider.core.conf.MapOperations; import org.apache.slider.core.launch.ContainerLauncher; import org.apache.slider.providers.ProviderRole; import org.apache.slider.providers.ProviderService; @@ -37,8 +33,8 @@ import org.apache.slider.server.appmaster.state.ContainerAssignment; import org.apache.slider.server.appmaster.state.RoleInstance; import org.apache.slider.server.appmaster.state.RoleStatus; -import org.apache.slider.server.services.workflow.WorkflowExecutorService; import org.apache.slider.server.services.workflow.ServiceThreadFactory; +import org.apache.slider.server.services.workflow.WorkflowExecutorService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -73,16 +69,6 @@ */ private final SliderFileSystem fs; - /** - * Path in the launch filesystem that refers to a configuration directory - * -the interpretation of it is left to the Provider - */ - private final Path generatedConfDirPath; - /** - * Path in the launch filesystem that refers to a temp directory - * which will be cleaned up at (some) time in the future - */ - private final Path launcherTmpDirPath; private Map envVars; @@ -91,21 +77,13 @@ * @param queueAccess * @param provider the provider * @param fs filesystem - * @param generatedConfDirPath path in the FS for the generated dir * @param envVars environment variables - * @param launcherTmpDirPath path for a temporary data in the launch process */ - public RoleLaunchService(QueueAccess queueAccess, - ProviderService provider, - SliderFileSystem fs, - Path generatedConfDirPath, - Map envVars, - Path launcherTmpDirPath) { + public RoleLaunchService(QueueAccess queueAccess, ProviderService provider, + SliderFileSystem fs, Map envVars) { super(ROLE_LAUNCH_SERVICE); this.actionQueue = queueAccess; this.fs = fs; - this.generatedConfDirPath = generatedConfDirPath; - this.launcherTmpDirPath = launcherTmpDirPath; this.provider = provider; this.envVars = envVars; } @@ -120,23 +98,13 @@ public void init(Configuration conf) { /** * Start an asychronous launch operation * @param assignment container assignment - * @param clusterSpec cluster spec to use for template * @param credentials credentials to use */ public void launchRole(ContainerAssignment assignment, - AggregateConf clusterSpec, - Credentials credentials) { - RoleStatus role = assignment.role; - String roleName = role.getName(); - String roleGroup = role.getGroup(); - // prelaunch safety check - Preconditions.checkArgument(provider.isSupportedRole(roleName)); + Application application, Credentials credentials) { RoleLaunchService.RoleLauncher launcher = - new RoleLaunchService.RoleLauncher(assignment, - clusterSpec, - clusterSpec.getResourceOperations().getOrAddComponent(roleGroup), - clusterSpec.getAppConfOperations().getOrAddComponent(roleGroup), - credentials); + new RoleLaunchService.RoleLauncher(assignment, application, + credentials); execute(launcher); } @@ -148,35 +116,21 @@ public void launchRole(ContainerAssignment assignment, private final ContainerAssignment assignment; // Allocated container public final Container container; - private final MapOperations resourceComponent; - private final MapOperations appComponent; - private final AggregateConf instanceDefinition; + public final Application application; public final ProviderRole role; private final Credentials credentials; - private Exception raisedException; public RoleLauncher(ContainerAssignment assignment, - AggregateConf instanceDefinition, - MapOperations resourceComponent, - MapOperations appComponent, + Application application, Credentials credentials) { this.assignment = assignment; this.credentials = credentials; this.container = assignment.container; RoleStatus roleStatus = assignment.role; - - assert resourceComponent != null; - assert appComponent != null; ProviderRole providerRole = roleStatus.getProviderRole(); - assert providerRole != null; this.role = providerRole; - this.resourceComponent = resourceComponent; - this.appComponent = appComponent; - this.instanceDefinition = instanceDefinition; - } + this.application = application; - public Exception getRaisedException() { - return raisedException; } @Override @@ -196,62 +150,46 @@ public void run() { containerLauncher.setupUGI(); containerLauncher.putEnv(envVars); - log.debug("Launching container {} into role {}", - container.getId(), - role.name); + log.info("Launching container {} into RoleName = {}, RoleGroup = {}", + container.getId(), role.name, role.group); - //now build up the configuration data - Path containerTmpDirPath = - new Path(launcherTmpDirPath, container.getId().toString()); - provider.buildContainerLaunchContext(containerLauncher, - instanceDefinition, - container, - role, - fs, - generatedConfDirPath, - resourceComponent, - appComponent, - containerTmpDirPath); + provider.buildContainerLaunchContext(containerLauncher, application, + container, role, fs); RoleInstance instance = new RoleInstance(container); String[] envDescription = containerLauncher.dumpEnvToString(); String commandsAsString = containerLauncher.getCommandsAsString(); - log.info("Starting container with command: {}", - commandsAsString); + log.info("Starting container with command: {}", commandsAsString); + instance.providerRole = role; instance.command = commandsAsString; instance.role = role.name; instance.group = role.group; instance.roleId = role.id; - instance.appVersion = instanceDefinition.getAppConfOperations() - .getGlobalOptions().get(SliderKeys.APP_VERSION); instance.environment = envDescription; - int delay = appComponent.getOptionInt( - AgentKeys.KEY_CONTAINER_LAUNCH_DELAY, 0); - int maxDelay = - getConfig().getInt(YarnConfiguration.RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS, - YarnConfiguration.DEFAULT_RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS); + long delay = role.component.getConfiguration() + .getPropertyLong(AgentKeys.KEY_CONTAINER_LAUNCH_DELAY, 0); + long maxDelay = getConfig() + .getLong(YarnConfiguration.RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS, + YarnConfiguration.DEFAULT_RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS); if (delay > maxDelay/1000) { log.warn("Container launch delay of {} exceeds the maximum allowed of" + " {} seconds. Delay will not be utilized.", delay, maxDelay/1000); delay = 0; } - log.info("Container launch delay for {} set to {} seconds", - role.name, delay); - actionQueue.schedule(new ActionStartContainer("starting " + role.name, - container, - containerLauncher.completeContainerLaunch(), - instance, - delay, - TimeUnit.SECONDS)); + log.info("Container launch delay for {} set to {} seconds", role.name, + delay); + actionQueue.schedule( + new ActionStartContainer("starting " + role.name, container, + containerLauncher.completeContainerLaunch(), instance, delay, + TimeUnit.SECONDS)); } catch (Exception e) { - log.error("Exception thrown while trying to start {}: {}", - role.name, e, e); - raisedException = e; + log.error("Exception thrown while trying to start " + role.name + + " container = " + container.getId() + " on host " + container + .getNodeId(), e); } } - } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java index f67ea58..c2fb2bc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java @@ -62,6 +62,7 @@ import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.client.api.AMRMClient; +import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync; import org.apache.hadoop.yarn.client.api.async.NMClientAsync; import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl; @@ -77,13 +78,12 @@ import org.apache.hadoop.yarn.webapp.WebAppException; import org.apache.hadoop.yarn.webapp.WebApps; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; -import org.apache.slider.api.ClusterDescription; import org.apache.slider.api.InternalKeys; import org.apache.slider.api.ResourceKeys; import org.apache.slider.api.RoleKeys; -import org.apache.slider.api.StatusKeys; +import org.apache.slider.api.proto.Messages; import org.apache.slider.api.proto.SliderClusterAPI; -import org.apache.slider.client.SliderYarnClientImpl; +import org.apache.slider.api.resource.Application; import org.apache.slider.common.SliderExitCodes; import org.apache.slider.common.SliderKeys; import org.apache.slider.common.params.AbstractActionArgs; @@ -95,10 +95,7 @@ import org.apache.slider.common.tools.SliderFileSystem; import org.apache.slider.common.tools.SliderUtils; import org.apache.slider.common.tools.SliderVersionInfo; -import org.apache.slider.core.buildutils.InstanceIO; import org.apache.slider.core.conf.AggregateConf; -import org.apache.slider.core.conf.ConfTree; -import org.apache.slider.core.conf.ConfTreeOperations; import org.apache.slider.core.conf.MapOperations; import org.apache.slider.core.exceptions.BadConfigException; import org.apache.slider.core.exceptions.SliderException; @@ -109,12 +106,13 @@ import org.apache.slider.core.main.LauncherExitCodes; import org.apache.slider.core.main.RunService; import org.apache.slider.core.main.ServiceLauncher; +import org.apache.slider.core.persist.JsonSerDeser; import org.apache.slider.core.registry.info.CustomRegistryConstants; +import org.apache.slider.providers.PlacementPolicy; import org.apache.slider.providers.ProviderCompleted; import org.apache.slider.providers.ProviderRole; import org.apache.slider.providers.ProviderService; import org.apache.slider.providers.SliderProviderFactory; -import org.apache.slider.providers.slideram.SliderAMClientProvider; import org.apache.slider.providers.slideram.SliderAMProviderService; import org.apache.slider.server.appmaster.actions.ActionHalt; import org.apache.slider.server.appmaster.actions.ActionRegisterServiceInstance; @@ -167,7 +165,6 @@ import java.io.File; import java.io.IOException; import java.net.InetSocketAddress; -import java.net.URI; import java.net.URL; import java.nio.ByteBuffer; import java.security.PrivilegedExceptionAction; @@ -252,7 +249,7 @@ /** * Credentials for propagating down to launched containers */ - private Credentials containerCredentials; + private Credentials containerCredentials = new Credentials(); /** * Slider IPC: Real service handler @@ -398,12 +395,14 @@ */ private boolean securityEnabled; private ContentCache contentCache; + private static final JsonSerDeser jsonSerDeser = + new JsonSerDeser(Application.class); /** * resource limits */ private Resource maximumResourceCapability; - + private Application application; /** * Service Constructor */ @@ -586,75 +585,27 @@ public Service initAndAddService(Service service) { /** * Create and run the cluster. - * @param clustername cluster name + * @param appName cluster name * @return exit code * @throws Throwable on a failure */ - private int createAndRunCluster(String clustername) throws Throwable { - - //load the cluster description from the cd argument - String sliderClusterDir = serviceArgs.getSliderClusterURI(); - URI sliderClusterURI = new URI(sliderClusterDir); - Path clusterDirPath = new Path(sliderClusterURI); - log.info("Application defined at {}", sliderClusterURI); + private int createAndRunCluster(String appName) throws Throwable { + Path appDir = new Path((serviceArgs.getSliderClusterURI())); SliderFileSystem fs = getClusterFS(); - - // build up information about the running application -this - // will be passed down to the cluster status - MapOperations appInformation = new MapOperations(); - - AggregateConf instanceDefinition = - InstanceIO.loadInstanceDefinitionUnresolved(fs, clusterDirPath); - instanceDefinition.setName(clustername); - - log.info("Deploying cluster {}:", instanceDefinition); - - // and resolve it - AggregateConf resolvedInstance = new AggregateConf( instanceDefinition); - resolvedInstance.resolve(); - - stateForProviders.setApplicationName(clustername); - + Path appJson = new Path(appDir, appName + ".json"); + application = jsonSerDeser + .load(fs.getFileSystem(), new Path(appDir, appName + ".json")); + log.info("Loaded application at " + appJson); + log.info("Application Json: " + application); + stateForProviders.setApplicationName(appName); Configuration serviceConf = getConfig(); - // extend AM configuration with component resource - MapOperations amConfiguration = resolvedInstance - .getAppConfOperations().getComponent(COMPONENT_AM); - // and patch configuration with prefix - if (amConfiguration != null) { - Map sliderAppConfKeys = amConfiguration.prefixedWith("slider."); - for (Map.Entry entry : sliderAppConfKeys.entrySet()) { - String k = entry.getKey(); - String v = entry.getValue(); - boolean exists = serviceConf.get(k) != null; - log.info("{} {} to {}", (exists ? "Overwriting" : "Setting"), k, v); - serviceConf.set(k, v); - } - } - - securityConfiguration = new SecurityConfiguration(serviceConf, resolvedInstance, clustername); // obtain security state - securityEnabled = securityConfiguration.isSecurityEnabled(); // set the global security flag for the instance definition - instanceDefinition.getAppConfOperations().set(KEY_SECURITY_ENABLED, securityEnabled); - // triggers resolution and snapshotting for agent - appState.setInitialInstanceDefinition(instanceDefinition); - - File confDir = getLocalConfDir(); - if (!confDir.exists() || !confDir.isDirectory()) { - log.info("Conf dir {} does not exist.", confDir); - File parentFile = confDir.getParentFile(); - log.info("Parent dir {}:\n{}", parentFile, SliderUtils.listDir(parentFile)); - } - //get our provider - MapOperations globalInternalOptions = getGlobalInternalOptions(); - String providerType = globalInternalOptions.getMandatoryOption( - InternalKeys.INTERNAL_PROVIDER_NAME); - log.info("Cluster provider type is {}", providerType); SliderProviderFactory factory = - SliderProviderFactory.createSliderProviderFactory(providerType); + SliderProviderFactory.createSliderProviderFactory("docker"); providerService = factory.createServerProvider(); // init the provider BUT DO NOT START IT YET initAndAddService(providerService); @@ -689,10 +640,6 @@ private int createAndRunCluster(String clustername) throws Throwable { ApplicationId appid = appAttemptID.getApplicationId(); log.info("AM for ID {}", appid.getId()); - appInformation.put(StatusKeys.INFO_AM_CONTAINER_ID, appMasterContainerID.toString()); - appInformation.put(StatusKeys.INFO_AM_APP_ID, appid.toString()); - appInformation.put(StatusKeys.INFO_AM_ATTEMPT_ID, appAttemptID.toString()); - Map envVars; List liveContainers; @@ -731,16 +678,14 @@ private int createAndRunCluster(String clustername) throws Throwable { } //bring up the Slider RPC service - buildPortScanner(instanceDefinition); - startSliderRPCServer(instanceDefinition); + buildPortScanner(); + startSliderRPCServer(); rpcServiceAddress = rpcService.getConnectAddress(); appMasterHostname = rpcServiceAddress.getAddress().getCanonicalHostName(); appMasterRpcPort = rpcServiceAddress.getPort(); appMasterTrackingUrl = null; log.info("AM Server is listening at {}:{}", appMasterHostname, appMasterRpcPort); - appInformation.put(StatusKeys.INFO_AM_HOSTNAME, appMasterHostname); - appInformation.set(StatusKeys.INFO_AM_RPC_PORT, appMasterRpcPort); log.info("Starting Yarn registry"); registryOperations = startRegistryOperationsService(); @@ -748,11 +693,11 @@ private int createAndRunCluster(String clustername) throws Throwable { //build the role map List providerRoles = new ArrayList<>(providerService.getRoles()); - providerRoles.addAll(SliderAMClientProvider.ROLES); + providerRoles.add(new ProviderRole(COMPONENT_AM, 0, + PlacementPolicy.EXCLUDE_FROM_FLEXING, + ResourceKeys.DEFAULT_NODE_FAILURE_THRESHOLD, 0, "")); // Start up the WebApp and track the URL for it - MapOperations component = instanceDefinition.getAppConfOperations() - .getComponent(SliderKeys.COMPONENT_AM); // Web service endpoints: initialize WebAppApiImpl webAppApi = @@ -760,9 +705,7 @@ private int createAndRunCluster(String clustername) throws Throwable { stateForProviders, providerService, registryOperations, metricsAndMonitoring, - actionQueues, - this, - contentCache); + actionQueues, contentCache); initAMFilterOptions(serviceConf); int webAppPort = deployWebApplication(webAppApi); @@ -770,9 +713,6 @@ private int createAndRunCluster(String clustername) throws Throwable { String scheme = WebAppUtils.HTTP_PREFIX; appMasterTrackingUrl = scheme + appMasterHostname + ":" + webAppPort; - appInformation.put(StatusKeys.INFO_AM_WEB_URL, appMasterTrackingUrl + "/"); - appInformation.set(StatusKeys.INFO_AM_WEB_PORT, webAppPort); - // ***************************************************** // Register self with ResourceManager // This will start heartbeating to the RM @@ -785,6 +725,7 @@ private int createAndRunCluster(String clustername) throws Throwable { appMasterTrackingUrl); maximumResourceCapability = amRegistrationData.getMaximumResourceCapability(); + //TODO should not read local configs !!! int minMemory = serviceConf.getInt(RM_SCHEDULER_MINIMUM_ALLOCATION_MB, DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB); // validate scheduler vcores allocation setting @@ -798,11 +739,7 @@ private int createAndRunCluster(String clustername) throws Throwable { // the max value as part of its lookup rmOperationHandler = new AsyncRMOperationHandler(asyncRMClient, maximumResourceCapability); - // set the RM-defined maximum cluster values - appInformation.put(ResourceKeys.YARN_CORES, Integer.toString(maxCores)); - appInformation.put(ResourceKeys.YARN_MEMORY, Integer.toString(maxMemory)); - - processAMCredentials(securityConfiguration); +// processAMCredentials(securityConfiguration); if (securityEnabled) { secretManager.setMasterKey( @@ -817,7 +754,9 @@ private int createAndRunCluster(String clustername) throws Throwable { // principal. Can do so now since AM registration with RM above required // tokens associated to principal String principal = securityConfiguration.getPrincipal(); - File localKeytabFile = securityConfiguration.getKeytabFile(instanceDefinition); + //TODO read key tab file from slider-am.xml + File localKeytabFile = + securityConfiguration.getKeytabFile(new AggregateConf()); // Now log in... login(principal, localKeytabFile); // obtain new FS reference that should be kerberos based and different @@ -829,10 +768,10 @@ private int createAndRunCluster(String clustername) throws Throwable { // YARN client. // Important: this is only valid at startup, and must be executed within // the right UGI context. Use with care. - SliderYarnClientImpl yarnClient = null; + YarnClient yarnClient = null; List nodeReports; try { - yarnClient = new SliderYarnClientImpl(); + yarnClient = YarnClient.createYarnClient(); yarnClient.init(getConfig()); yarnClient.start(); nodeReports = getNodeReports(yarnClient); @@ -857,37 +796,22 @@ private int createAndRunCluster(String clustername) throws Throwable { liveContainers = amRegistrationData.getContainersFromPreviousAttempts(); - //now validate the installation - Configuration providerConf = - providerService.loadProviderConfigurationInformation(confDir); - - providerService.initializeApplicationConfiguration(instanceDefinition, - fs, null); - - providerService.validateApplicationConfiguration(instanceDefinition, - confDir, - securityEnabled); //determine the location for the role history data - Path historyDir = new Path(clusterDirPath, HISTORY_DIR_NAME); + Path historyDir = new Path(appDir, HISTORY_DIR_NAME); //build the instance AppStateBindingInfo binding = new AppStateBindingInfo(); - binding.instanceDefinition = instanceDefinition; binding.serviceConfig = serviceConf; - binding.publishedProviderConf = providerConf; binding.roles = providerRoles; binding.fs = fs.getFileSystem(); binding.historyPath = historyDir; binding.liveContainers = liveContainers; - binding.applicationInfo = appInformation; binding.releaseSelector = providerService.createContainerReleaseSelector(); binding.nodeReports = nodeReports; + binding.application = application; appState.buildInstance(binding); - providerService.rebuildContainerDetails(liveContainers, - instanceDefinition.getName(), appState.getRolePriorityMap()); - // add the AM to the list of nodes in the cluster appState.buildAppMasterNode(appMasterContainerID, @@ -908,8 +832,8 @@ private int createAndRunCluster(String clustername) throws Throwable { } String rolesTmpSubdir = appMasterContainerID.toString() + "/roles"; - String amTmpDir = globalInternalOptions.getMandatoryOption(InternalKeys.INTERNAL_AM_TMP_DIR); - + String amTmpDir = "/tmp"; + //TODO read tmpDir from slider-am.xml Path tmpDirPath = new Path(amTmpDir); Path launcherTmpDirPath = new Path(tmpDirPath, rolesTmpSubdir); fs.getFileSystem().mkdirs(launcherTmpDirPath); @@ -917,10 +841,7 @@ private int createAndRunCluster(String clustername) throws Throwable { //launcher service launchService = new RoleLaunchService(actionQueues, providerService, - fs, - new Path(getGeneratedConfDir()), - envVars, - launcherTmpDirPath); + fs, envVars); deployChildService(launchService); @@ -932,14 +853,7 @@ private int createAndRunCluster(String clustername) throws Throwable { sliderAMProvider.bind(stateForProviders, actionQueues, liveContainers); // chaos monkey - maybeStartMonkey(); - - // setup token renewal and expiry handling for long lived apps -// if (!securityConfiguration.isKeytabProvided() && -// SliderUtils.isHadoopClusterSecure(getConfig())) { -// fsDelegationTokenManager = new FsDelegationTokenManager(actionQueues); -// fsDelegationTokenManager.acquireDelegationToken(getConfig()); -// } +// maybeStartMonkey(); // if not a secure cluster, extract the username -it will be // propagated to workers @@ -955,12 +869,12 @@ private int createAndRunCluster(String clustername) throws Throwable { log.info("Application Master Initialization Completed"); initCompleted.set(true); - scheduleFailureWindowResets(instanceDefinition.getResources()); - scheduleEscalation(instanceDefinition.getInternal()); + scheduleFailureWindowResets(application.getConfiguration()); + scheduleEscalation(application.getConfiguration()); try { // schedule YARN Registry registration - queue(new ActionRegisterServiceInstance(clustername, appid)); + queue(new ActionRegisterServiceInstance(appName, appid, application)); // log the YARN and web UIs log.info("RM Webapp address {}", @@ -973,7 +887,7 @@ private int createAndRunCluster(String clustername) throws Throwable { // launch the real provider; this is expected to trigger a callback that // starts the node review process - launchProviderService(instanceDefinition, confDir); + launchProviderService(); // start handling any scheduled events @@ -1000,7 +914,7 @@ private int createAndRunCluster(String clustername) throws Throwable { * @throws InterruptedException */ private ApplicationAttemptReport getApplicationAttemptReport( - final SliderYarnClientImpl yarnClient) + final YarnClient yarnClient) throws YarnException, IOException, InterruptedException { Preconditions.checkNotNull(yarnClient, "Null Yarn client"); ApplicationAttemptReport report; @@ -1019,14 +933,14 @@ public ApplicationAttemptReport run() throws Exception { } /** - * List the node reports: uses {@link SliderYarnClientImpl} as the login user + * List the node reports: uses {@link YarnClient} as the login user * @param yarnClient client to the RM * @return the node reports * @throws IOException * @throws YarnException * @throws InterruptedException */ - private List getNodeReports(final SliderYarnClientImpl yarnClient) + private List getNodeReports(final YarnClient yarnClient) throws IOException, YarnException, InterruptedException { Preconditions.checkNotNull(yarnClient, "Null Yarn client"); List nodeReports; @@ -1051,7 +965,7 @@ public ApplicationAttemptReport run() throws Exception { * Creates and starts the web application, and adds a * WebAppService service under the AM, to ensure * a managed web application shutdown. - * @param webAppApi web app API instance + * @param webAppApi web application API instance * @return port the web application is deployed on * @throws IOException general problems starting the webapp (network, etc) * @throws WebAppException other issues @@ -1117,12 +1031,14 @@ private void processAMCredentials(SecurityConfiguration securityConfig) /** * Build up the port scanner. This may include setting a port range. */ - private void buildPortScanner(AggregateConf instanceDefinition) + private void buildPortScanner() throws BadConfigException { portScanner = new PortScanner(); - String portRange = instanceDefinition. - getAppConfOperations().getGlobalOptions(). - getOption(SliderKeys.KEY_ALLOWED_PORT_RANGE, "0"); + String portRange = "0"; + //TODO read from slider-am.xml +// String portRange = instanceDefinition. +// getAppConfOperations().getGlobalOptions(). +// getOption(SliderKeys.KEY_ALLOWED_PORT_RANGE, "0"); if (!"0".equals(portRange)) { portScanner.setPortRange(portRange); } @@ -1203,7 +1119,7 @@ private void initAMFilterOptions(Configuration serviceConf) { * @throws IOException */ public void registerServiceInstance(String instanceName, - ApplicationId appId) throws IOException { + ApplicationId appId, Application application) throws IOException { // the registry is running, so register services @@ -1241,9 +1157,8 @@ public void registerServiceInstance(String instanceName, // set any provided attributes - setProvidedServiceRecordAttributes( - getInstanceDefinition().getAppConfOperations().getComponent( - SliderKeys.COMPONENT_AM), serviceRecord); + setUserProvidedServiceRecordAttributes(application.getConfiguration(), + serviceRecord); // register the service's entry log.info("Service Record \n{}", serviceRecord); @@ -1276,7 +1191,7 @@ protected void setupInitialRegistryPaths() throws IOException { /** * Handler for {@link RegisterComponentInstance action} - * Register/re-register an ephemeral container that is already in the app state + * Register/re-register an ephemeral container that is already in the application state * @param id the component * @param description component description * @param type component type @@ -1296,9 +1211,8 @@ public boolean registerComponent(ContainerId id, String description, container.description = description; container.set(YarnRegistryAttributes.YARN_PERSISTENCE, PersistencePolicies.CONTAINER); - MapOperations compOps = getInstanceDefinition().getAppConfOperations(). - getComponent(type); - setProvidedServiceRecordAttributes(compOps, container); + setUserProvidedServiceRecordAttributes( + instance.providerRole.component.getConfiguration(), container); try { yarnRegistryOperations.putComponent(cid, container); } catch (IOException e) { @@ -1309,14 +1223,12 @@ public boolean registerComponent(ContainerId id, String description, return true; } - protected void setProvidedServiceRecordAttributes(MapOperations ops, - ServiceRecord record) { + protected void setUserProvidedServiceRecordAttributes( + org.apache.slider.api.resource.Configuration conf, ServiceRecord record) { String prefix = RoleKeys.SERVICE_RECORD_ATTRIBUTE_PREFIX; - for (Map.Entry entry : ops.entrySet()) { - if (entry.getKey().startsWith( - prefix)) { - String key = entry.getKey().substring( - prefix.length() + 1); + for (Map.Entry entry : conf.getProperties().entrySet()) { + if (entry.getKey().startsWith(prefix)) { + String key = entry.getKey().substring(prefix.length() + 1); record.set(key, entry.getValue().trim()); } } @@ -1366,35 +1278,6 @@ private void checkAndWarnForAuthTokenProblems() { } /** - * Build the configuration directory passed in or of the target FS - * @return the file - */ - public File getLocalConfDir() { - File confdir = - new File(SliderKeys.PROPAGATED_CONF_DIR_NAME).getAbsoluteFile(); - return confdir; - } - - /** - * Get the path to the DFS configuration that is defined in the cluster specification - * @return the generated configuration dir - */ - public String getGeneratedConfDir() { - return getGlobalInternalOptions().get( - InternalKeys.INTERNAL_GENERATED_CONF_PATH); - } - - /** - * Get the global internal options for the AM - * @return a map to access the internals - */ - public MapOperations getGlobalInternalOptions() { - return getInstanceDefinition() - .getInternalOperations(). - getGlobalOptions(); - } - - /** * Get the filesystem of this cluster * @return the FS of the config */ @@ -1500,7 +1383,7 @@ private synchronized int finish() throws Exception { launchService.stop(); //now release all containers - releaseAllContainers(); + releaseAllContainers(application); // When the application completes, it should send a finish application // signal to the RM @@ -1536,7 +1419,7 @@ public Object getProxy(Class protocol, InetSocketAddress addr) { /** * Start the slider RPC server */ - private void startSliderRPCServer(AggregateConf instanceDefinition) + private void startSliderRPCServer() throws IOException, SliderException { verifyIPCAccess(); @@ -1612,16 +1495,8 @@ public void onContainersAllocated(List allocatedContainers) { //for each assignment: instantiate that role for (ContainerAssignment assignment : assignments) { - try { - launchService.launchRole(assignment, getInstanceDefinition(), - buildContainerCredentials()); - } catch (IOException e) { - // Can be caused by failure to renew credentials with the remote - // service. If so, don't launch the application. Container is retained, - // though YARN will take it away after a timeout. - log.error("Failed to build credentials to launch container: {}", e, e); - - } + //TODO Do we need to pass credentials to containers? + launchService.launchRole(assignment, application, null); } //for all the operations, exec them @@ -1724,22 +1599,14 @@ public synchronized void onUpgradeContainers( * Implementation of cluster flexing. * It should be the only way that anything -even the AM itself on startup- * asks for nodes. - * @param resources the resource tree * @throws SliderException slider problems, including invalid configs * @throws IOException IO problems */ - public void flexCluster(ConfTree resources) + public void flexCluster(Messages.FlexComponentRequestProto request) throws IOException, SliderException { - - AggregateConf newConf = - new AggregateConf(appState.getInstanceDefinitionSnapshot()); - newConf.setResources(resources); - // verify the new definition is valid - sliderAMProvider.validateInstanceDefinition(newConf); - providerService.validateInstanceDefinition(newConf); - - appState.updateResourceDefinitions(resources); - + if (request != null) { + appState.updateComponents(request); + } // reset the scheduled windows...the values // may have changed appState.resetFailureCounts(); @@ -1750,24 +1617,37 @@ public void flexCluster(ConfTree resources) /** * Schedule the failure window - * @param resources the resource tree * @throws BadConfigException if the window is out of range */ - private void scheduleFailureWindowResets(ConfTree resources) throws - BadConfigException { + private void scheduleFailureWindowResets( + org.apache.slider.api.resource.Configuration conf) { + ResetFailureWindow reset = new ResetFailureWindow(rmOperationHandler); - ConfTreeOperations ops = new ConfTreeOperations(resources); - MapOperations globals = ops.getGlobalOptions(); - long seconds = globals.getTimeRange(ResourceKeys.CONTAINER_FAILURE_WINDOW, - ResourceKeys.DEFAULT_CONTAINER_FAILURE_WINDOW_DAYS, - ResourceKeys.DEFAULT_CONTAINER_FAILURE_WINDOW_HOURS, - ResourceKeys.DEFAULT_CONTAINER_FAILURE_WINDOW_MINUTES, 0); - if (seconds > 0) { - log.info( - "Scheduling the failure window reset interval to every {} seconds", - seconds); - RenewingAction renew = new RenewingAction<>( - reset, seconds, seconds, TimeUnit.SECONDS, 0); + + long days = + conf.getPropertyLong(ResourceKeys.CONTAINER_FAILURE_WINDOW + ".days", + ResourceKeys.DEFAULT_CONTAINER_FAILURE_WINDOW_DAYS); + long hours = + conf.getPropertyLong(ResourceKeys.CONTAINER_FAILURE_WINDOW + ".hours", + ResourceKeys.DEFAULT_CONTAINER_FAILURE_WINDOW_HOURS); + long minutes = + conf.getPropertyLong(ResourceKeys.CONTAINER_FAILURE_WINDOW + ".minutes", + ResourceKeys.DEFAULT_CONTAINER_FAILURE_WINDOW_MINUTES); + long seconds = + conf.getPropertyLong(ResourceKeys.CONTAINER_FAILURE_WINDOW + ".seconds", + 0); + Preconditions + .checkState(days >= 0 && hours >= 0 && minutes >= 0 && seconds >= 0, + "Time range for has negative time component %s:%s:%s:%s", days, + hours, minutes, seconds); + long totalMinutes = days * 24 * 60 + hours * 24 + minutes; + long totalSeconds = totalMinutes * 60 + seconds; + if (totalSeconds > 0) { + log.info("Scheduling the failure window reset interval to every {}" + + " seconds", totalSeconds); + RenewingAction renew = + new RenewingAction<>(reset, totalSeconds, totalSeconds, + TimeUnit.SECONDS, 0); actionQueues.renewing("failures", renew); } else { log.info("Failure window reset interval is not set"); @@ -1776,16 +1656,15 @@ private void scheduleFailureWindowResets(ConfTree resources) throws /** * Schedule the escalation action - * @param internal * @throws BadConfigException */ - private void scheduleEscalation(ConfTree internal) throws BadConfigException { + private void scheduleEscalation( + org.apache.slider.api.resource.Configuration conf) { EscalateOutstandingRequests escalate = new EscalateOutstandingRequests(); - ConfTreeOperations ops = new ConfTreeOperations(internal); - int seconds = ops.getGlobalOptions().getOptionInt(InternalKeys.ESCALATION_CHECK_INTERVAL, + long seconds = conf.getPropertyLong(InternalKeys.ESCALATION_CHECK_INTERVAL, InternalKeys.DEFAULT_ESCALATION_CHECK_INTERVAL); - RenewingAction renew = new RenewingAction<>( - escalate, seconds, seconds, TimeUnit.SECONDS, 0); + RenewingAction renew = + new RenewingAction<>(escalate, seconds, seconds, TimeUnit.SECONDS, 0); actionQueues.renewing("escalation", renew); } @@ -1810,6 +1689,7 @@ public void handleReviewAndFlexApplicationSize(ReviewAndFlexApplicationSize acti if ( actionQueues.hasQueuedActionWithAttribute( AsyncAction.ATTR_REVIEWS_APP_SIZE | AsyncAction.ATTR_HALTS_APP)) { + //TODO Loop all actions to check duplicate ?? // this operation isn't needed at all -existing duplicate or shutdown due return; } @@ -1861,11 +1741,11 @@ public void escalateOutstandingRequests() { /** * Shutdown operation: release all containers */ - private void releaseAllContainers() { + private void releaseAllContainers(Application application) { // Add the sleep here (before releasing containers) so that applications get // time to perform graceful shutdown try { - long timeout = getContainerReleaseTimeout(); + long timeout = getContainerReleaseTimeout(application); if (timeout > 0) { Thread.sleep(timeout); } @@ -1879,16 +1759,11 @@ private void releaseAllContainers() { } } - private long getContainerReleaseTimeout() { + private long getContainerReleaseTimeout(Application application) { // Get container release timeout in millis or 0 if the property is not set. - // If non-zero then add the agent heartbeat delay time, since it can take up - // to that much time for agents to receive the stop command. - int timeout = getInstanceDefinition().getAppConfOperations() - .getGlobalOptions() - .getOptionInt(SliderKeys.APP_CONTAINER_RELEASE_TIMEOUT, 0); - if (timeout > 0) { - timeout += SliderKeys.APP_CONTAINER_HEARTBEAT_INTERVAL_SEC; - } + long timeout = application.getConfiguration() + .getPropertyLong(SliderKeys.APP_CONTAINER_RELEASE_TIMEOUT, 0); + // convert to millis long timeoutInMillis = timeout * 1000l; log.info("Container release timeout in millis = {}", timeoutInMillis); @@ -2000,27 +1875,15 @@ public void updateBlacklist(List blacklistAdditions, /** * Launch the provider service - * - * @param instanceDefinition definition of the service - * @param confDir directory of config data * @throws IOException * @throws SliderException */ - protected synchronized void launchProviderService(AggregateConf instanceDefinition, - File confDir) - throws IOException, SliderException { - Map env = new HashMap<>(); - boolean execStarted = providerService.exec(instanceDefinition, confDir, env, - this); - if (execStarted) { - providerService.registerServiceListener(this); - providerService.start(); - } else { - // didn't start, so don't register - providerService.start(); - // and send the started event ourselves - eventCallbackEvent(null); - } + protected synchronized void launchProviderService() + throws IOException, SliderException { + // didn't start, so don't register + providerService.start(); + // and send the started event ourselves + eventCallbackEvent(null); } /* =================================================================== */ @@ -2033,7 +1896,7 @@ public void eventCallbackEvent(Object parameter) { appState.noteAMLive(); // now ask for the cluster nodes try { - flexCluster(getInstanceDefinition().getResources()); + flexCluster(null); } catch (Exception e) { // cluster flex failure: log log.error("Failed to flex cluster nodes: {}", e, e); @@ -2221,16 +2084,6 @@ public void onStopContainerError(ContainerId containerId, Throwable t) { LOG_YARN.warn("Failed to stop Container {}", containerId); } - public AggregateConf getInstanceDefinition() { - return appState.getInstanceDefinition(); - } - - /** - * This is the status, the live model - */ - public ClusterDescription getClusterDescription() { - return appState.getClusterStatus(); - } public ProviderService getProviderService() { return providerService; @@ -2278,12 +2131,12 @@ public void onExceptionInThread(Thread thread, Throwable exception) { } /** - * Start the chaos monkey + * TODO Start the chaos monkey * @return true if it started */ private boolean maybeStartMonkey() { - MapOperations internals = getGlobalInternalOptions(); - +// MapOperations internals = getGlobalInternalOptions(); + MapOperations internals = new MapOperations(); Boolean enabled = internals.getOptionBool(InternalKeys.CHAOS_MONKEY_ENABLED, InternalKeys.DEFAULT_CHAOS_MONKEY_ENABLED); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionFlexCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionFlexCluster.java index 6b61681..a660958 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionFlexCluster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionFlexCluster.java @@ -18,6 +18,7 @@ package org.apache.slider.server.appmaster.actions; +import org.apache.slider.api.proto.Messages; import org.apache.slider.core.conf.ConfTree; import org.apache.slider.server.appmaster.SliderAppMaster; import org.apache.slider.server.appmaster.state.AppState; @@ -26,19 +27,16 @@ public class ActionFlexCluster extends AsyncAction { - public final ConfTree resources; - - public ActionFlexCluster(String name, - long delay, - TimeUnit timeUnit, ConfTree resources) { + final Messages.FlexComponentRequestProto requestProto; + public ActionFlexCluster(String name, long delay, TimeUnit timeUnit, + Messages.FlexComponentRequestProto requestProto) { super(name, delay, timeUnit, ATTR_CHANGES_APP_SIZE); - this.resources = resources; + this.requestProto = requestProto; } - @Override public void execute(SliderAppMaster appMaster, QueueAccess queueService, AppState appState) throws Exception { - appMaster.flexCluster(resources); + appMaster.flexCluster(requestProto); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionRegisterServiceInstance.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionRegisterServiceInstance.java index ca330af..0d7f7d4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionRegisterServiceInstance.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionRegisterServiceInstance.java @@ -19,6 +19,7 @@ package org.apache.slider.server.appmaster.actions; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.slider.api.resource.Application; import org.apache.slider.server.appmaster.SliderAppMaster; import org.apache.slider.server.appmaster.state.AppState; @@ -31,21 +32,13 @@ private final String instanceName; private final ApplicationId appId; - + private final Application application; public ActionRegisterServiceInstance(String instanceName, - ApplicationId appId) { + ApplicationId appId, Application application) { super("ActionRegisterServiceInstance"); this.instanceName = instanceName; this.appId = appId; - } - - public ActionRegisterServiceInstance(String instanceName, - ApplicationId appId, - long delay, - TimeUnit timeUnit) { - super("ActionRegisterServiceInstance", delay, timeUnit); - this.instanceName = instanceName; - this.appId = appId; + this.application = application; } @Override @@ -54,6 +47,6 @@ public void execute(SliderAppMaster appMaster, AppState appState) throws Exception { // YARN Registry do the registration - appMaster.registerServiceInstance(instanceName, appId); + appMaster.registerServiceInstance(instanceName, appId, application); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolPBImpl.java index fbd408e..4d483c7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolPBImpl.java @@ -70,11 +70,12 @@ public long getProtocolVersion(String protocol, long clientVersion) } @Override - public Messages.FlexClusterResponseProto flexCluster(RpcController controller, - Messages.FlexClusterRequestProto request) throws ServiceException { + public Messages.FlexComponentResponseProto flexComponent( + RpcController controller, Messages.FlexComponentRequestProto request) + throws ServiceException { try { - return real.flexCluster(request); - } catch (Exception e) { + return real.flexComponent(request); + } catch (IOException e) { throw wrap(e); } } @@ -90,19 +91,6 @@ public long getProtocolVersion(String protocol, long clientVersion) } } - - @Override - public Messages.GetInstanceDefinitionResponseProto getInstanceDefinition( - RpcController controller, - Messages.GetInstanceDefinitionRequestProto request) - throws ServiceException { - try { - return real.getInstanceDefinition(request); - } catch (Exception e) { - throw wrap(e); - } - } - @Override public Messages.ListNodeUUIDsByRoleResponseProto listNodeUUIDsByRole( RpcController controller, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolProxy.java index 448c6f3..c60d609 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolProxy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolProxy.java @@ -110,10 +110,10 @@ private IOException convert(ServiceException se) { } @Override - public Messages.FlexClusterResponseProto flexCluster(Messages.FlexClusterRequestProto request) - throws IOException { + public Messages.FlexComponentResponseProto flexComponent( + Messages.FlexComponentRequestProto request) throws IOException { try { - return endpoint.flexCluster(NULL_CONTROLLER, request); + return endpoint.flexComponent(NULL_CONTROLLER, request); } catch (ServiceException e) { throw convert(e); } @@ -131,19 +131,6 @@ private IOException convert(ServiceException se) { } } - - @Override - public Messages.GetInstanceDefinitionResponseProto getInstanceDefinition( - Messages.GetInstanceDefinitionRequestProto request) throws - IOException, - YarnException { - try { - return endpoint.getInstanceDefinition(NULL_CONTROLLER, request); - } catch (ServiceException e) { - throw convert(e); - } - } - @Override public Messages.ListNodeUUIDsByRoleResponseProto listNodeUUIDsByRole(Messages.ListNodeUUIDsByRoleRequestProto request) throws IOException, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderIPCService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderIPCService.java index 70c2f05..32ca942 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderIPCService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderIPCService.java @@ -27,6 +27,7 @@ import org.apache.slider.api.ClusterDescription; import org.apache.slider.api.SliderClusterProtocol; import org.apache.slider.api.proto.Messages; +import org.apache.slider.api.resource.Application; import org.apache.slider.api.types.ApplicationLivenessInformation; import org.apache.slider.api.types.ComponentInformation; import org.apache.slider.api.types.ContainerInformation; @@ -38,6 +39,7 @@ import org.apache.slider.core.main.LauncherExitCodes; import org.apache.slider.core.persist.AggregateConfSerDeser; import org.apache.slider.core.persist.ConfTreeSerDeser; +import org.apache.slider.core.persist.JsonSerDeser; import org.apache.slider.server.appmaster.AppMasterActionOperations; import org.apache.slider.server.appmaster.actions.ActionFlexCluster; import org.apache.slider.server.appmaster.actions.ActionHalt; @@ -78,6 +80,9 @@ private final MetricsAndMonitoring metricsAndMonitoring; private final AppMasterActionOperations amOperations; private final ContentCache cache; + private static final JsonSerDeser jsonSerDeser = + new JsonSerDeser(Application.class); + /** * This is the prefix used for metrics @@ -195,17 +200,12 @@ public void queue(AsyncAction action) { return Messages.UpgradeContainersResponseProto.getDefaultInstance(); } - @Override //SliderClusterProtocol - public Messages.FlexClusterResponseProto flexCluster(Messages.FlexClusterRequestProto request) - throws IOException { + @Override + public Messages.FlexComponentResponseProto flexComponent( + Messages.FlexComponentRequestProto request) throws IOException { onRpcCall("flex"); - String payload = request.getClusterSpec(); - ConfTreeSerDeser confTreeSerDeser = new ConfTreeSerDeser(); - ConfTree updatedResources = confTreeSerDeser.fromJson(payload); - schedule(new ActionFlexCluster("flex", 1, TimeUnit.MILLISECONDS, - updatedResources)); - return Messages.FlexClusterResponseProto.newBuilder().setResponse( - true).build(); + schedule(new ActionFlexCluster("flex", 1, TimeUnit.MILLISECONDS, request)); + return Messages.FlexComponentResponseProto.newBuilder().build(); } @Override //SliderClusterProtocol @@ -216,38 +216,10 @@ public void queue(AsyncAction action) { String result; //quick update //query and json-ify - ClusterDescription cd = state.refreshClusterStatus(); - result = cd.toJsonString(); - String stat = result; + Application application = state.refreshClusterStatus(); + String stat = jsonSerDeser.toJson(application); return Messages.GetJSONClusterStatusResponseProto.newBuilder() - .setClusterSpec(stat) - .build(); - } - - @Override - public Messages.GetInstanceDefinitionResponseProto getInstanceDefinition( - Messages.GetInstanceDefinitionRequestProto request) - throws IOException, YarnException { - - onRpcCall("getinstancedefinition"); - String internal; - String resources; - String app; - AggregateConf instanceDefinition = - state.getInstanceDefinitionSnapshot(); - internal = instanceDefinition.getInternal().toJson(); - resources = instanceDefinition.getResources().toJson(); - app = instanceDefinition.getAppConf().toJson(); - assert internal != null; - assert resources != null; - assert app != null; - log.debug("Generating getInstanceDefinition Response"); - Messages.GetInstanceDefinitionResponseProto.Builder builder = - Messages.GetInstanceDefinitionResponseProto.newBuilder(); - builder.setInternal(internal); - builder.setResources(resources); - builder.setApplication(app); - return builder.build(); + .setClusterSpec(stat).build(); } @Override //SliderClusterProtocol diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/security/SecurityConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/security/SecurityConfiguration.java index 9a89c39..b31babc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/security/SecurityConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/security/SecurityConfiguration.java @@ -138,6 +138,7 @@ public boolean isKeytabProvided() { public File getKeytabFile(AggregateConf instanceDefinition) throws SliderException, IOException { + //TODO implement this for dash semantic String keytabFullPath = instanceDefinition.getAppConfOperations() .getComponent(SliderKeys.COMPONENT_AM) .get(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java index 6f54959..8fd57b4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java @@ -38,21 +38,19 @@ import org.apache.hadoop.yarn.util.resource.Resources; import org.apache.slider.api.ClusterDescription; import org.apache.slider.api.ClusterDescriptionKeys; -import org.apache.slider.api.ClusterDescriptionOperations; import org.apache.slider.api.ClusterNode; import org.apache.slider.api.InternalKeys; -import org.apache.slider.api.ResourceKeys; import org.apache.slider.api.StatusKeys; +import org.apache.slider.api.proto.Messages; +import org.apache.slider.api.resource.Application; +import org.apache.slider.api.resource.ApplicationState; +import org.apache.slider.api.resource.Component; import org.apache.slider.api.types.ApplicationLivenessInformation; import org.apache.slider.api.types.ComponentInformation; import org.apache.slider.api.types.RoleStatistics; import org.apache.slider.common.SliderExitCodes; import org.apache.slider.common.SliderKeys; -import org.apache.slider.common.tools.ConfigHelper; import org.apache.slider.common.tools.SliderUtils; -import org.apache.slider.core.conf.AggregateConf; -import org.apache.slider.core.conf.ConfTree; -import org.apache.slider.core.conf.ConfTreeOperations; import org.apache.slider.core.conf.MapOperations; import org.apache.slider.core.exceptions.BadClusterStateException; import org.apache.slider.core.exceptions.BadConfigException; @@ -60,9 +58,6 @@ import org.apache.slider.core.exceptions.NoSuchNodeException; import org.apache.slider.core.exceptions.SliderInternalStateException; import org.apache.slider.core.exceptions.TriggerClusterTeardownException; -import org.apache.slider.core.persist.AggregateConfSerDeser; -import org.apache.slider.core.persist.ConfTreeSerDeser; -import org.apache.slider.providers.PlacementPolicy; import org.apache.slider.providers.ProviderRole; import org.apache.slider.server.appmaster.management.LongGauge; import org.apache.slider.server.appmaster.management.MetricsAndMonitoring; @@ -77,7 +72,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -92,9 +86,7 @@ import static org.apache.slider.api.ResourceKeys.*; import static org.apache.slider.api.RoleKeys.*; import static org.apache.slider.api.StateValues.*; -import static org.apache.slider.providers.docker.DockerKeys.DEFAULT_DOCKER_USE_PRIVILEGED; -import static org.apache.slider.providers.docker.DockerKeys.DOCKER_IMAGE; -import static org.apache.slider.providers.docker.DockerKeys.DOCKER_USE_PRIVILEGED; +import static org.apache.slider.api.resource.ApplicationState.STARTED; /** * The model of all the ongoing state of a Slider AM. @@ -117,42 +109,7 @@ */ private boolean applicationLive = false; - /** - * The definition of the instance. Flexing updates the resources section - * This is used as a synchronization point on activities that update - * the CD, and also to update some of the structures that - * feed in to the CD - */ - private AggregateConf instanceDefinition; - - /** - * Time the instance definition snapshots were created - */ - private long snapshotTime; - - /** - * Snapshot of the instance definition. This is fully - * resolved. - */ - private AggregateConf instanceDefinitionSnapshot; - - /** - * Snapshot of the raw instance definition; unresolved and - * without any patch of an AM into it. - */ - private AggregateConf unresolvedInstanceDefinition; - - /** - * snapshot of resources as of last update time - */ - private ConfTreeOperations resourcesSnapshot; - private ConfTreeOperations appConfSnapshot; - private ConfTreeOperations internalsSnapshot; - - /** - * This is the status, the live model - */ - private ClusterDescription clusterStatus = new ClusterDescription(); + private Application app; /** * Metadata provided by the AM for use in filling in status requests @@ -160,12 +117,6 @@ private Map applicationInfo; /** - * Client properties created via the provider -static for the life - * of the application - */ - private Map clientProperties = new HashMap<>(); - - /** * This is a template of the cluster status */ private ClusterDescription clusterStatusTemplate = new ClusterDescription(); @@ -396,47 +347,13 @@ public AtomicInteger getCompletionOfUnknownContainerEvent() { /** * Get the current view of the cluster status. - *

- * Calls to {@link #refreshClusterStatus()} trigger a - * refresh of this field. - *

* This is read-only * to the extent that changes here do not trigger updates in the * application state. * @return the cluster status */ - public synchronized ClusterDescription getClusterStatus() { - return clusterStatus; - } - - @VisibleForTesting - protected synchronized void setClusterStatus(ClusterDescription clusterDesc) { - this.clusterStatus = clusterDesc; - } - - /** - * Set the instance definition -this also builds the (now obsolete) - * cluster specification from it. - * - * Important: this is for early binding and must not be used after the build - * operation is complete. - * @param definition initial definition - * @throws BadConfigException - */ - public synchronized void setInitialInstanceDefinition(AggregateConf definition) - throws BadConfigException, IOException { - log.debug("Setting initial instance definition"); - // snapshot the definition - AggregateConfSerDeser serDeser = new AggregateConfSerDeser(); - - unresolvedInstanceDefinition = serDeser.fromInstance(definition); - - this.instanceDefinition = serDeser.fromInstance(definition); - onInstanceDefinitionUpdated(); - } - - public synchronized AggregateConf getInstanceDefinition() { - return instanceDefinition; + public synchronized Application getClusterStatus() { + return app; } /** @@ -475,58 +392,24 @@ public void setContainerLimits(int minMemory,int maxMemory, int minCores, int ma maxResource = recordFactory.newResource(containerMaxMemory, containerMaxCores); } - public ConfTreeOperations getResourcesSnapshot() { - return resourcesSnapshot; - } - - public ConfTreeOperations getAppConfSnapshot() { - return appConfSnapshot; - } - - public ConfTreeOperations getInternalsSnapshot() { - return internalsSnapshot; - } - public boolean isApplicationLive() { return applicationLive; } - public long getSnapshotTime() { - return snapshotTime; - } - - public synchronized AggregateConf getInstanceDefinitionSnapshot() { - return instanceDefinitionSnapshot; - } - - public AggregateConf getUnresolvedInstanceDefinition() { - return unresolvedInstanceDefinition; - } public synchronized void buildInstance(AppStateBindingInfo binding) throws BadClusterStateException, BadConfigException, IOException { binding.validate(); log.debug("Building application state"); - publishedProviderConf = binding.publishedProviderConf; applicationInfo = binding.applicationInfo != null ? binding.applicationInfo : new HashMap(); - clientProperties = new HashMap<>(); containerReleaseSelector = binding.releaseSelector; - - Set confKeys = ConfigHelper.sortedConfigKeys(publishedProviderConf); - - // Add the -site configuration properties - for (String key : confKeys) { - String val = publishedProviderConf.get(key); - clientProperties.put(key, val); - } - // set the cluster specification (once its dependency the client properties // is out the way - setInitialInstanceDefinition(binding.instanceDefinition); + this.app = binding.application; //build the initial role list List roleList = new ArrayList<>(binding.roles); @@ -534,42 +417,33 @@ public synchronized void buildInstance(AppStateBindingInfo binding) buildRole(providerRole); } - ConfTreeOperations resources = instanceDefinition.getResourceOperations(); - - Set roleNames = resources.getComponentNames(); - for (String name : roleNames) { + int priority = 1; + for (Component component : app.getComponents()) { + String name = component.getName(); if (roles.containsKey(name)) { continue; } - if (hasUniqueNames(resources, name)) { - log.info("Skipping group {}", name); + if (component.getUniqueComponentSupport()) { + log.info("Skipping group " + name + ", as it's unique component"); continue; } - // this is a new value - log.info("Adding role {}", name); - MapOperations resComponent = resources.getComponent(name); - ProviderRole dynamicRole = createDynamicProviderRole(name, resComponent); + log.info("Adding component: " + name); + ProviderRole dynamicRole = + createComponent(name, name, component, priority++); buildRole(dynamicRole); roleList.add(dynamicRole); } //then pick up the requirements buildRoleRequirementsFromResources(); - //set the livespan - MapOperations globalResOpts = instanceDefinition.getResourceOperations().getGlobalOptions(); - - startTimeThreshold = globalResOpts.getOptionInt( - InternalKeys.INTERNAL_CONTAINER_FAILURE_SHORTLIFE, - InternalKeys.DEFAULT_INTERNAL_CONTAINER_FAILURE_SHORTLIFE); - - failureThreshold = globalResOpts.getOptionInt( - CONTAINER_FAILURE_THRESHOLD, + org.apache.slider.api.resource.Configuration conf = app.getConfiguration(); + startTimeThreshold = + conf.getPropertyLong(InternalKeys.INTERNAL_CONTAINER_FAILURE_SHORTLIFE, + InternalKeys.DEFAULT_INTERNAL_CONTAINER_FAILURE_SHORTLIFE); + failureThreshold = (int) conf.getPropertyLong(CONTAINER_FAILURE_THRESHOLD, DEFAULT_CONTAINER_FAILURE_THRESHOLD); - nodeFailureThreshold = globalResOpts.getOptionInt( - NODE_FAILURE_THRESHOLD, + nodeFailureThreshold = (int) conf.getPropertyLong(NODE_FAILURE_THRESHOLD, DEFAULT_NODE_FAILURE_THRESHOLD); - initClusterStatus(); - // set up the role history roleHistory = new RoleHistory(roleStatusMap.values(), recordFactory); @@ -578,7 +452,6 @@ public synchronized void buildInstance(AppStateBindingInfo binding) // trigger first node update roleHistory.onNodesUpdated(binding.nodeReports); - //rebuild any live containers rebuildModelFromRestart(binding.liveContainers); @@ -586,180 +459,55 @@ public synchronized void buildInstance(AppStateBindingInfo binding) logServerURL = binding.serviceConfig.get(YarnConfiguration.YARN_LOG_SERVER_URL, ""); //mark as live applicationLive = true; + app.setState(STARTED); } - public void initClusterStatus() { - //copy into cluster status. - ClusterDescription status = ClusterDescription.copy(clusterStatusTemplate); - status.state = STATE_CREATED; - MapOperations infoOps = new MapOperations("info", status.info); - infoOps.mergeWithoutOverwrite(applicationInfo); - SliderUtils.addBuildInfo(infoOps, "status"); + //TODO WHY do we need to create the component for AM ? + public ProviderRole createComponent(String name, String group, + Component component, int priority) throws BadConfigException { - long now = now(); - status.setInfoTime(StatusKeys.INFO_LIVE_TIME_HUMAN, - StatusKeys.INFO_LIVE_TIME_MILLIS, - now); - SliderUtils.setInfoTime(infoOps, - StatusKeys.INFO_LIVE_TIME_HUMAN, - StatusKeys.INFO_LIVE_TIME_MILLIS, - now); - if (0 == status.createTime) { - status.createTime = now; - SliderUtils.setInfoTime(infoOps, - StatusKeys.INFO_CREATE_TIME_HUMAN, - StatusKeys.INFO_CREATE_TIME_MILLIS, - now); - } - status.state = STATE_LIVE; + org.apache.slider.api.resource.Configuration conf = + component.getConfiguration(); + long placementTimeout = conf.getPropertyLong(PLACEMENT_ESCALATE_DELAY, + DEFAULT_PLACEMENT_ESCALATE_DELAY_SECONDS); + int threshold = (int) conf + .getPropertyLong(NODE_FAILURE_THRESHOLD, nodeFailureThreshold); + ProviderRole newRole = + new ProviderRole(name, group, priority, 0, threshold, + placementTimeout, "", component); - //set the app state to this status - setClusterStatus(status); - } - - /** - * Build a dynamic provider role - * @param name name of role - * @return a new provider role - * @throws BadConfigException bad configuration - */ - public ProviderRole createDynamicProviderRole(String name, MapOperations component) - throws BadConfigException { - return createDynamicProviderRole(name, name, component); - } - - /** - * Build a dynamic provider role - * @param name name of role - * @param group group of role - * @return a new provider role - * @throws BadConfigException bad configuration - */ - public ProviderRole createDynamicProviderRole(String name, String group, MapOperations component) - throws BadConfigException { - String priOpt = component.getMandatoryOption(COMPONENT_PRIORITY); - int priority = SliderUtils.parseAndValidate( - "value of " + name + " " + COMPONENT_PRIORITY, priOpt, 0, 1, -1); - - String placementOpt = component.getOption(COMPONENT_PLACEMENT_POLICY, - Integer.toString(PlacementPolicy.DEFAULT)); - - int placement = SliderUtils.parseAndValidate( - "value of " + name + " " + COMPONENT_PLACEMENT_POLICY, placementOpt, 0, 0, -1); - - int placementTimeout = component.getOptionInt(PLACEMENT_ESCALATE_DELAY, - DEFAULT_PLACEMENT_ESCALATE_DELAY_SECONDS); - - ProviderRole newRole = new ProviderRole(name, - group, - priority, - placement, - getNodeFailureThresholdForRole(group), - placementTimeout, - component.getOption(YARN_LABEL_EXPRESSION, DEF_YARN_LABEL_EXPRESSION)); - log.info("New {} ", newRole); + log.info("Created a new role " + newRole); return newRole; } - /** - * Actions to perform when an instance definition is updated - * Currently: - *

    - *
  1. - * resolve the configuration - *
  2. - *
  3. - * update the cluster spec derivative - *
  4. - *
- * - * @throws BadConfigException - */ - private synchronized void onInstanceDefinitionUpdated() - throws BadConfigException, IOException { - - log.debug("Instance definition updated"); - //note the time - snapshotTime = now(); - - for (String component : instanceDefinition.getResourceOperations().getComponentNames()) { - instanceDefinition.getAppConfOperations().getOrAddComponent(component); - } - - // resolve references if not already done - instanceDefinition.resolve(); - - // force in the AM desired state values - ConfTreeOperations resources = instanceDefinition.getResourceOperations(); - - if (resources.getComponent(SliderKeys.COMPONENT_AM) != null) { - resources.setComponentOpt( - SliderKeys.COMPONENT_AM, COMPONENT_INSTANCES, "1"); - } - - - //snapshot all three sectons - resourcesSnapshot = ConfTreeOperations.fromInstance(instanceDefinition.getResources()); - appConfSnapshot = ConfTreeOperations.fromInstance(instanceDefinition.getAppConf()); - internalsSnapshot = ConfTreeOperations.fromInstance(instanceDefinition.getInternal()); - //build a new aggregate from the snapshots - instanceDefinitionSnapshot = new AggregateConf(resourcesSnapshot.confTree, - appConfSnapshot.confTree, - internalsSnapshot.confTree); - instanceDefinitionSnapshot.setName(instanceDefinition.getName()); - - clusterStatusTemplate = ClusterDescriptionOperations.buildFromInstanceDefinition( - instanceDefinition); - - // Add the -site configuration properties - for (Map.Entry prop : clientProperties.entrySet()) { - clusterStatusTemplate.clientProperties.put(prop.getKey(), prop.getValue()); + public synchronized void updateComponents( + Messages.FlexComponentRequestProto requestProto) + throws BadConfigException { + for (Component component : app.getComponents()) { + if (component.getName().equals(requestProto.getName())) { + component + .setNumberOfContainers((long) requestProto.getNumberOfContainers()); + } } - - } - - /** - * The resource configuration is updated -review and update state. - * @param resources updated resources specification - * @return a list of any dynamically added provider roles - * (purely for testing purposes) - */ - @VisibleForTesting - public synchronized List updateResourceDefinitions(ConfTree resources) - throws BadConfigException, IOException { - log.debug("Updating resources to {}", resources); - // snapshot the (possibly unresolved) values - ConfTreeSerDeser serDeser = new ConfTreeSerDeser(); - unresolvedInstanceDefinition.setResources( - serDeser.fromInstance(resources)); - // assign another copy under the instance definition for resolving - // and then driving application size - instanceDefinition.setResources(serDeser.fromInstance(resources)); - onInstanceDefinitionUpdated(); - - // propagate the role table - Map> updated = resources.components; - getClusterStatus().roles = SliderUtils.deepClone(updated); - getClusterStatus().updateTime = now(); - return buildRoleRequirementsFromResources(); + //TODO update cluster description + buildRoleRequirementsFromResources(); } /** * build the role requirements from the cluster specification * @return a list of any dynamically added provider roles */ - private List buildRoleRequirementsFromResources() throws BadConfigException { + private List buildRoleRequirementsFromResources() + throws BadConfigException { List newRoles = new ArrayList<>(0); // now update every role's desired count. // if there are no instance values, that role count goes to zero - - ConfTreeOperations resources = - instanceDefinition.getResourceOperations(); - // Add all the existing roles + // component name -> number of containers Map groupCounts = new HashMap<>(); + for (RoleStatus roleStatus : getRoleStatusMap().values()) { if (roleStatus.isExcludeFromFlexing()) { // skip inflexible roles, e.g AM itself @@ -768,10 +516,11 @@ private synchronized void onInstanceDefinitionUpdated() long currentDesired = roleStatus.getDesired(); String role = roleStatus.getName(); String roleGroup = roleStatus.getGroup(); - int desiredInstanceCount = getDesiredInstanceCount(resources, roleGroup); + Component component = roleStatus.getProviderRole().component; + int desiredInstanceCount = component.getNumberOfContainers().intValue(); int newDesired = desiredInstanceCount; - if (hasUniqueNames(resources, roleGroup)) { + if (component.getUniqueComponentSupport()) { Integer groupCount = 0; if (groupCounts.containsKey(roleGroup)) { groupCount = groupCounts.get(roleGroup); @@ -799,50 +548,47 @@ private synchronized void onInstanceDefinitionUpdated() // now the dynamic ones. Iterate through the the cluster spec and // add any role status entries not in the role status - Set roleNames = resources.getComponentNames(); - for (String name : roleNames) { + + List list = new ArrayList<>(getRoleStatusMap().values()); + for (RoleStatus roleStatus : list) { + String name = roleStatus.getName(); + Component component = roleStatus.getProviderRole().component; if (roles.containsKey(name)) { continue; } - if (hasUniqueNames(resources, name)) { + if (component.getUniqueComponentSupport()) { // THIS NAME IS A GROUP - int desiredInstanceCount = getDesiredInstanceCount(resources, name); + int desiredInstanceCount = component.getNumberOfContainers().intValue(); Integer groupCount = 0; if (groupCounts.containsKey(name)) { groupCount = groupCounts.get(name); } for (int i = groupCount + 1; i <= desiredInstanceCount; i++) { - int priority = resources.getComponentOptInt(name, COMPONENT_PRIORITY, i); + int priority = roleStatus.getPriority(); // this is a new instance of an existing group String newName = String.format("%s%d", name, i); int newPriority = getNewPriority(priority + i - 1); log.info("Adding new role {}", newName); - MapOperations component = resources.getComponent(name, - Collections.singletonMap(COMPONENT_PRIORITY, - Integer.toString(newPriority))); - if (component == null) { - throw new BadConfigException("Component is null for name = " + name - + ", newPriority =" + newPriority); - } - ProviderRole dynamicRole = createDynamicProviderRole(newName, name, component); - RoleStatus roleStatus = buildRole(dynamicRole); - roleStatus.setDesired(1); - log.info("New role {}", roleStatus); + ProviderRole dynamicRole = + createComponent(newName, name, component, newPriority); + RoleStatus newRole = buildRole(dynamicRole); + newRole.setDesired(1); + log.info("New role {}", newRole); if (roleHistory != null) { - roleHistory.addNewRole(roleStatus); + roleHistory.addNewRole(newRole); } newRoles.add(dynamicRole); } } else { // this is a new value log.info("Adding new role {}", name); - MapOperations component = resources.getComponent(name); - ProviderRole dynamicRole = createDynamicProviderRole(name, component); - RoleStatus roleStatus = buildRole(dynamicRole); - roleStatus.setDesired(getDesiredInstanceCount(resources, name)); - log.info("New role {}", roleStatus); + ProviderRole dynamicRole = + createComponent(name, name, component, roleStatus.getPriority()); + RoleStatus newRole = buildRole(dynamicRole); + newRole.setDesired(component.getNumberOfContainers()); + log.info("New role {}", newRole); if (roleHistory != null) { - roleHistory.addNewRole(roleStatus); + roleHistory.addNewRole(newRole); } newRoles.add(dynamicRole); } @@ -861,37 +607,6 @@ private int getNewPriority(int start) { } /** - * Get the desired instance count of a role, rejecting negative values - * @param resources resource map - * @param roleGroup role group - * @return the instance count - * @throws BadConfigException if the count is negative - */ - private int getDesiredInstanceCount(ConfTreeOperations resources, - String roleGroup) throws BadConfigException { - int desiredInstanceCount = - resources.getComponentOptInt(roleGroup, COMPONENT_INSTANCES, 0); - - if (desiredInstanceCount < 0) { - log.error("Role {} has negative desired instances : {}", roleGroup, - desiredInstanceCount); - throw new BadConfigException( - "Negative instance count (%) requested for component %s", - desiredInstanceCount, roleGroup); - } - return desiredInstanceCount; - } - - private Boolean hasUniqueNames(ConfTreeOperations resources, String group) { - MapOperations component = resources.getComponent(group); - if (component == null) { - log.info("Component was null for {} when checking unique names", group); - return Boolean.FALSE; - } - return component.getOptionBool(UNIQUE_NAMES, Boolean.FALSE); - } - - /** * Add knowledge of a role. * This is a build-time operation that is not synchronized, and * should be used while setting up the system state -before servicing @@ -923,8 +638,7 @@ public RoleStatus buildRole(ProviderRole providerRole) throws BadConfigException */ private void buildRoleResourceRequirements() { for (RoleStatus role : roleStatusMap.values()) { - role.setResourceRequirements( - buildResourceRequirements(role, recordFactory.newResource())); + role.setResourceRequirements(buildResourceRequirements(role)); } } @@ -1353,61 +1067,33 @@ private void decOutstandingContainerRequests() { } } - - /** - * Get the value of a YARN requirement (cores, RAM, etc). - * These are returned as integers, but there is special handling of the - * string {@link ResourceKeys#YARN_RESOURCE_MAX}, which triggers - * the return of the maximum value. - * @param group component to get from - * @param option option name - * @param defVal default value - * @param maxVal value to return if the max val is requested - * @return parsed value - * @throws NumberFormatException if the role could not be parsed. - */ - private int getResourceRequirement(ConfTreeOperations resources, - String group, - String option, - int defVal, - int maxVal) { - - String val = resources.getComponentOpt(group, option, - Integer.toString(defVal)); - Integer intVal; - if (YARN_RESOURCE_MAX.equals(val)) { - intVal = maxVal; - } else { - intVal = Integer.decode(val); - } - return intVal; - } - /** * Build up the resource requirements for this role from the * cluster specification, including substituing max allowed values * if the specification asked for it. * @param role role - * @param capability capability to set up. A new one may be created * during normalization */ - public Resource buildResourceRequirements(RoleStatus role, Resource capability) { + public Resource buildResourceRequirements(RoleStatus role) { // Set up resource requirements from role values String name = role.getName(); - String group = role.getGroup(); - ConfTreeOperations resources = getResourcesSnapshot(); - int cores = getResourceRequirement(resources, - group, - YARN_CORES, - DEF_YARN_CORES, - containerMaxCores); - capability.setVirtualCores(cores); - int ram = getResourceRequirement(resources, group, - YARN_MEMORY, - DEF_YARN_MEMORY, - containerMaxMemory); - capability.setMemory(ram); - log.debug("Component {} has RAM={}, vCores ={}", name, ram, cores); + Component component = role.getProviderRole().component; + if (component == null) { + // this is for AM container + // TODO why do we need to create the component for AM ? + return Resource.newInstance(1, 512); + } + int cores = Math.min(containerMaxCores, component.getResource().getCpus()); + if (cores <= 0) { + cores = DEF_YARN_CORES; + } + long mem = Math.min(containerMaxMemory, + Long.parseLong(component.getResource().getMemory())); + if (mem <= 0) { + mem = DEF_YARN_MEMORY; + } + Resource capability = Resource.newInstance(mem, cores); + log.debug("Component {} has RAM={}, vCores ={}", name, mem, cores); Resource normalized = recordFactory.normalize(capability, minResource, maxResource); if (!Resources.equals(normalized, capability)) { @@ -1771,29 +1457,32 @@ public synchronized float getApplicationProgressPercentage() { return percentage; } - /** - * Update the cluster description with the current application state - */ - public ClusterDescription refreshClusterStatus() { - return refreshClusterStatus(null); + public synchronized Application refreshAppStatus() { + //TODO replace ClusterDescription with Application + related statistics + + return app; } /** * Update the cluster description with the current application state - * @param providerStatus status from the provider for the cluster info section */ - public synchronized ClusterDescription refreshClusterStatus(Map providerStatus) { + + public synchronized Application refreshClusterStatus() { + + //TODO replace ClusterDescription with Application + related statistics + //TODO build container stats + app.setState(ApplicationState.STARTED); + return app; + /* + return app; + ClusterDescription cd = getClusterStatus(); long now = now(); cd.setInfoTime(StatusKeys.INFO_STATUS_TIME_HUMAN, StatusKeys.INFO_STATUS_TIME_MILLIS, now); - if (providerStatus != null) { - for (Map.Entry entry : providerStatus.entrySet()) { - cd.setInfo(entry.getKey(), entry.getValue()); - } - } + MapOperations infoOps = new MapOperations("info", cd.info); infoOps.mergeWithoutOverwrite(applicationInfo); SliderUtils.addBuildInfo(infoOps, "status"); @@ -1810,32 +1499,8 @@ public synchronized ClusterDescription refreshClusterStatus(Map cd.status = new HashMap<>(); cd.status.put(ClusterDescriptionKeys.KEY_CLUSTER_LIVE, clusterNodes); - for (RoleStatus role : getRoleStatusMap().values()) { String rolename = role.getName(); - if (hasUniqueNames(instanceDefinition.getResourceOperations(), - role.getGroup())) { - cd.setRoleOpt(rolename, COMPONENT_PRIORITY, role.getPriority()); - cd.setRoleOpt(rolename, ROLE_GROUP, role.getGroup()); - MapOperations groupOptions = instanceDefinition.getResourceOperations() - .getComponent(role.getGroup()); - SliderUtils.mergeMapsIgnoreDuplicateKeys(cd.getRole(rolename), - groupOptions.options); - } - String prefix = instanceDefinition.getAppConfOperations() - .getComponentOpt(role.getGroup(), ROLE_PREFIX, null); - if (SliderUtils.isSet(prefix)) { - cd.setRoleOpt(rolename, ROLE_PREFIX, SliderUtils.trimPrefix(prefix)); - } - String dockerImage = instanceDefinition.getAppConfOperations() - .getComponentOpt(role.getGroup(), DOCKER_IMAGE, null); - if (SliderUtils.isSet(dockerImage)) { - cd.setRoleOpt(rolename, DOCKER_IMAGE, dockerImage); - Boolean dockerUsePrivileged = instanceDefinition.getAppConfOperations() - .getComponentOptBool(role.getGroup(), DOCKER_USE_PRIVILEGED, - DEFAULT_DOCKER_USE_PRIVILEGED); - cd.setRoleOpt(rolename, DOCKER_USE_PRIVILEGED, dockerUsePrivileged); - } List instances = instanceMap.get(rolename); int nodeCount = instances != null ? instances.size(): 0; cd.setRoleOpt(rolename, COMPONENT_INSTANCES, @@ -1861,7 +1526,7 @@ public synchronized ClusterDescription refreshClusterStatus(Map // liveness cd.liveness = getApplicationLivenessInformation(); - return cd; + return cd;*/ } /** @@ -2000,26 +1665,10 @@ private void checkFailureThreshold(RoleStatus role) * @return the threshold for failures */ private int getFailureThresholdForRole(RoleStatus roleStatus) { - ConfTreeOperations resources = - instanceDefinition.getResourceOperations(); - return resources.getComponentOptInt(roleStatus.getGroup(), - CONTAINER_FAILURE_THRESHOLD, - failureThreshold); + return (int) roleStatus.getProviderRole().component.getConfiguration() + .getPropertyLong(NODE_FAILURE_THRESHOLD, nodeFailureThreshold); } - /** - * Get the node failure threshold for a specific role, falling back to - * the global one if not - * @param roleGroup role group - * @return the threshold for failures - */ - private int getNodeFailureThresholdForRole(String roleGroup) { - ConfTreeOperations resources = - instanceDefinition.getResourceOperations(); - return resources.getComponentOptInt(roleGroup, - NODE_FAILURE_THRESHOLD, - nodeFailureThreshold); - } /** * Reset the "recent" failure counts of all roles @@ -2329,13 +1978,12 @@ private RoleInstance findRoleInstanceOnHost(NodeInstance node, int roleId) { * @param assignments the assignments of roles to containers * @param operations any allocation or release operations */ - public synchronized void onContainersAllocated(List allocatedContainers, - List assignments, - List operations) { - assignments.clear(); - operations.clear(); + public synchronized void onContainersAllocated( + List allocatedContainers, + List assignments, + List operations) { List ordered = roleHistory.prepareAllocationList(allocatedContainers); - log.debug("onContainersAllocated(): Total containers allocated = {}", ordered.size()); + log.info("onContainersAllocated(): Total containers allocated = {}", ordered.size()); for (Container container : ordered) { final NodeId nodeId = container.getNodeId(); String containerHostInfo = nodeId.getHost() + ":" + nodeId.getPort(); @@ -2437,8 +2085,7 @@ private boolean rebuildModelFromRestart(List liveContainers) for (Container container : liveContainers) { addRestartedContainer(container); } - clusterStatus.setInfo(StatusKeys.INFO_CONTAINERS_AM_RESTART, - Integer.toString(liveContainers.size())); + app.setNumberOfRunningContainers((long)liveContainers.size()); return true; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java index a8aa1a2..0a18985 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java @@ -24,7 +24,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.NodeReport; -import org.apache.slider.core.conf.AggregateConf; +import org.apache.slider.api.resource.Application; import org.apache.slider.providers.ProviderRole; import java.util.ArrayList; @@ -38,9 +38,8 @@ * are added. */ public class AppStateBindingInfo { - public AggregateConf instanceDefinition; public Configuration serviceConfig = new Configuration(); - public Configuration publishedProviderConf = new Configuration(false); + public Application application = null; public List roles = new ArrayList<>(); public FileSystem fs; public Path historyPath; @@ -51,9 +50,7 @@ public List nodeReports = new ArrayList<>(0); public void validate() throws IllegalArgumentException { - Preconditions.checkArgument(instanceDefinition != null, "null instanceDefinition"); Preconditions.checkArgument(serviceConfig != null, "null appmasterConfig"); - Preconditions.checkArgument(publishedProviderConf != null, "null publishedProviderConf"); Preconditions.checkArgument(releaseSelector != null, "null releaseSelector"); Preconditions.checkArgument(roles != null, "null providerRoles"); Preconditions.checkArgument(fs != null, "null fs"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java index 37e9a7f..bf638a5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java @@ -23,6 +23,7 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.slider.api.ClusterDescription; import org.apache.slider.api.ClusterNode; +import org.apache.slider.api.resource.Application; import org.apache.slider.api.types.ApplicationLivenessInformation; import org.apache.slider.api.types.ComponentInformation; import org.apache.slider.api.types.NodeInformation; @@ -130,46 +131,16 @@ public PublishedConfigSet getOrCreatePublishedConfigSet(String name) { } @Override - public ClusterDescription getClusterStatus() { + public Application getApplication() { return appState.getClusterStatus(); } @Override - public ConfTreeOperations getResourcesSnapshot() { - return appState.getResourcesSnapshot(); - } - - @Override - public ConfTreeOperations getAppConfSnapshot() { - return appState.getAppConfSnapshot(); - } - - @Override - public ConfTreeOperations getInternalsSnapshot() { - return appState.getInternalsSnapshot(); - } - - @Override public boolean isApplicationLive() { return appState.isApplicationLive(); } @Override - public long getSnapshotTime() { - return appState.getSnapshotTime(); - } - - @Override - public AggregateConf getInstanceDefinitionSnapshot() { - return appState.getInstanceDefinitionSnapshot(); - } - - @Override - public AggregateConf getUnresolvedInstanceDefinition() { - return appState.getUnresolvedInstanceDefinition(); - } - - @Override public RoleStatus lookupRoleStatus(int key) { return appState.lookupRoleStatus(key); } @@ -221,7 +192,7 @@ public RoleInstance getLiveInstanceByContainerID(String containerId) throws } @Override - public ClusterDescription refreshClusterStatus() { + public Application refreshClusterStatus() { return appState.refreshClusterStatus(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java index 30cfec9..de52f4e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java @@ -29,6 +29,7 @@ import org.apache.slider.api.proto.Messages; import org.apache.slider.api.types.ContainerInformation; import org.apache.slider.common.tools.SliderUtils; +import org.apache.slider.providers.ProviderRole; import java.util.ArrayList; import java.util.Arrays; @@ -40,6 +41,7 @@ public final class RoleInstance implements Cloneable { public Container container; + public ProviderRole providerRole; /** * Container ID */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java index ad91183..0d618a6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java @@ -24,6 +24,7 @@ import org.apache.slider.api.ClusterDescription; import org.apache.slider.api.ClusterNode; import org.apache.slider.api.StatusKeys; +import org.apache.slider.api.resource.Application; import org.apache.slider.api.types.ApplicationLivenessInformation; import org.apache.slider.api.types.ComponentInformation; import org.apache.slider.api.types.NodeInformation; @@ -105,29 +106,7 @@ * Get the current cluster description * @return the actual state of the cluster */ - ClusterDescription getClusterStatus(); - - /** - * Get at the snapshot of the resource config - * Changes here do not affect the application state. - * @return the most recent settings - */ - ConfTreeOperations getResourcesSnapshot(); - - /** - * Get at the snapshot of the appconf config - * Changes here do not affect the application state. - * @return the most recent settings - */ - ConfTreeOperations getAppConfSnapshot(); - - /** - * Get at the snapshot of the internals config. - * Changes here do not affect the application state. - * @return the internals settings - */ - - ConfTreeOperations getInternalsSnapshot(); + Application getApplication(); /** * Flag set to indicate the application is live -this only happens @@ -135,22 +114,8 @@ */ boolean isApplicationLive(); - long getSnapshotTime(); - /** - * Get a snapshot of the entire aggregate configuration - * @return the aggregate configuration - */ - AggregateConf getInstanceDefinitionSnapshot(); - - /** - * Get the desired/unresolved value - * @return unresolved - */ - AggregateConf getUnresolvedInstanceDefinition(); - - /** - * Look up a role from its key -or fail + * Look up a role from its key -or fail * * @param key key to resolve * @return the status @@ -159,7 +124,7 @@ RoleStatus lookupRoleStatus(int key); /** - * Look up a role from its key -or fail + * Look up a role from its key -or fail * * @param c container in a role * @return the status @@ -232,7 +197,7 @@ RoleInstance getLiveInstanceByContainerID(String containerId) /** * Update the cluster description with anything interesting */ - ClusterDescription refreshClusterStatus(); + Application refreshClusterStatus(); /** * Get a deep clone of the role status list. Concurrent events may mean this diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java index d20f1ad..03248cf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java @@ -18,7 +18,6 @@ import org.apache.hadoop.registry.client.api.RegistryOperations; import org.apache.slider.providers.ProviderService; -import org.apache.slider.server.appmaster.AppMasterActionOperations; import org.apache.slider.server.appmaster.actions.QueueAccess; import org.apache.slider.server.appmaster.management.MetricsAndMonitoring; import org.apache.slider.server.appmaster.state.StateAccessForProviders; @@ -39,14 +38,12 @@ private final RegistryOperations registryOperations; private final MetricsAndMonitoring metricsAndMonitoring; private final QueueAccess queues; - private final AppMasterActionOperations appMasterOperations; private final ContentCache contentCache; public WebAppApiImpl(StateAccessForProviders appState, ProviderService provider, RegistryOperations registryOperations, MetricsAndMonitoring metricsAndMonitoring, QueueAccess queues, - AppMasterActionOperations appMasterOperations, ContentCache contentCache) { - this.appMasterOperations = appMasterOperations; + ContentCache contentCache) { this.contentCache = contentCache; checkNotNull(appState); checkNotNull(provider); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWebServices.java index aed87d8..60004af 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWebServices.java @@ -18,6 +18,7 @@ import com.google.inject.Inject; import com.google.inject.Singleton; +import org.apache.slider.api.resource.Application; import org.apache.slider.server.appmaster.web.WebAppApi; import org.apache.slider.server.appmaster.web.rest.application.ApplicationResource; import org.apache.slider.server.appmaster.web.rest.management.ManagementResource; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java index 52068d6..c13646a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java @@ -22,6 +22,7 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.webapp.BadRequestException; import org.apache.hadoop.yarn.webapp.NotFoundException; +import org.apache.slider.api.resource.Application; import org.apache.slider.api.types.ApplicationLivenessInformation; import org.apache.slider.api.types.ComponentInformation; import org.apache.slider.api.types.ContainerInformation; @@ -107,6 +108,15 @@ public ApplicationResource(WebAppApi slider) { actionQueues = slider.getQueues(); } + @GET + @Path("/") + @Produces({APPLICATION_JSON}) + public Application getApplication() { + markGet(SLIDER_SUBPATH_APPLICATION); + return slider.getAppState().getApplication(); + } + + /** * Build a new JSON-marshallable list of string elements * @param elements elements @@ -116,14 +126,6 @@ public ApplicationResource(WebAppApi slider) { return Lists.newArrayList(elements); } - @GET - @Path("/") - @Produces({APPLICATION_JSON}) - public List getRoot() { - markGet(SLIDER_SUBPATH_APPLICATION); - return ROOT_ENTRIES; - } - /** * Enum model values: desired and resolved * @return the desired and resolved model @@ -160,58 +162,6 @@ public ConfTree getModelDesiredResources() { return lookupConfTree(MODEL_DESIRED_RESOURCES); } -/* - @PUT - @Path(MODEL_DESIRED_RESOURCES) -// @Consumes({APPLICATION_JSON, TEXT_PLAIN}) - @Consumes({TEXT_PLAIN}) - @Produces({APPLICATION_JSON}) -*/ - public ConfTree setModelDesiredResources( - String json) { - markPut(SLIDER_SUBPATH_APPLICATION, MODEL_DESIRED_RESOURCES); - int size = json != null ? json.length() : 0; - log.info("PUT {} {} bytes:\n{}", MODEL_DESIRED_RESOURCES, - size, - json); - if (size == 0) { - log.warn("No JSON in PUT request; rejecting"); - throw new BadRequestException("No JSON in PUT"); - } - - try { - ConfTreeSerDeser serDeser = new ConfTreeSerDeser(); - ConfTree updated = serDeser.fromJson(json); - queue(new ActionFlexCluster("flex", - 1, TimeUnit.MILLISECONDS, - updated)); - // return the updated value, even though it potentially hasn't yet - // been executed - return updated; - } catch (Exception e) { - throw buildException("PUT to "+ MODEL_DESIRED_RESOURCES , e); - } - } - @PUT - @Path(MODEL_DESIRED_RESOURCES) - @Consumes({APPLICATION_JSON}) - @Produces({APPLICATION_JSON}) - public ConfTree setModelDesiredResources( - ConfTree updated) { - try { - queue(new ActionFlexCluster("flex", - 1, TimeUnit.MILLISECONDS, - updated)); - // return the updated value, even though it potentially hasn't yet - // been executed - return updated; - } catch (Exception e) { - throw buildException("PUT to "+ MODEL_DESIRED_RESOURCES , e); - } - } - - - @GET @Path(MODEL_RESOLVED) @Produces({APPLICATION_JSON}) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/AggregateModelRefresher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/AggregateModelRefresher.java index ee28abf..261e66e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/AggregateModelRefresher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/AggregateModelRefresher.java @@ -23,7 +23,6 @@ /** * Refresh the aggregate desired model via - * {@link StateAccessForProviders#getInstanceDefinitionSnapshot()} */ public class AggregateModelRefresher implements ResourceRefresher { @@ -39,9 +38,6 @@ public AggregateModelRefresher(StateAccessForProviders state, @Override public AggregateConf refresh() throws Exception { - return - resolved ? - state.getInstanceDefinitionSnapshot() - : state.getUnresolvedInstanceDefinition(); + return new AggregateConf(); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/AppconfRefresher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/AppconfRefresher.java index 06460cc..190a51e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/AppconfRefresher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/AppconfRefresher.java @@ -44,10 +44,7 @@ public AppconfRefresher(StateAccessForProviders state, @Override public ConfTree refresh() throws Exception { - AggregateConf aggregateConf = - unresolved ? - state.getUnresolvedInstanceDefinition(): - state.getInstanceDefinitionSnapshot(); + AggregateConf aggregateConf = new AggregateConf(); ConfTree ct = resources ? aggregateConf.getResources() : aggregateConf.getAppConf(); return new ConfTreeSerDeser().fromInstance(ct); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveResourcesRefresher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveResourcesRefresher.java index f988297..a554af4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveResourcesRefresher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveResourcesRefresher.java @@ -37,8 +37,8 @@ public LiveResourcesRefresher(StateAccessForProviders state) { @Override public ConfTree refresh() throws Exception { - // snapshot resources - ConfTreeOperations resources = state.getResourcesSnapshot(); + // TODO + ConfTreeOperations resources = new ConfTreeOperations(new ConfTree()); // then add actual values Map roleStatusMap = state.getRoleStatusMap(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ResourceSnapshotRefresher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ResourceSnapshotRefresher.java deleted file mode 100644 index c16912a..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ResourceSnapshotRefresher.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.web.rest.application.resources; - -import org.apache.slider.core.conf.ConfTree; -import org.apache.slider.core.conf.ConfTreeOperations; -import org.apache.slider.server.appmaster.state.StateAccessForProviders; - -public class ResourceSnapshotRefresher implements ResourceRefresher { - - private final StateAccessForProviders state; - - public ResourceSnapshotRefresher(StateAccessForProviders state) { - this.state = state; - } - - @Override - public ConfTree refresh() throws Exception { - - // snapshot resources - ConfTreeOperations resources = state.getResourcesSnapshot(); - return resources.getConfTree(); - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/ManagementResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/ManagementResource.java index f27711a..14d9400 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/ManagementResource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/ManagementResource.java @@ -88,6 +88,7 @@ public ConfTreeResource getConfTreeResource(@PathParam(CONFIG) String config, } protected AggregateConf getAggregateConf() { - return slider.getAppState().getInstanceDefinitionSnapshot(); + //TODO + return new AggregateConf(); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ClusterSpecificationBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ClusterSpecificationBlock.java index 2f02f27..79b687f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ClusterSpecificationBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ClusterSpecificationBlock.java @@ -49,7 +49,7 @@ protected void doRender(Hamlet html) { * @return */ private String getJson() { - return appState.getClusterStatus().toString(); + return appState.getApplication().toString(); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ContainerStatsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ContainerStatsBlock.java index 8b7d695..4796d6c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ContainerStatsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ContainerStatsBlock.java @@ -26,8 +26,8 @@ import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TR; -import org.apache.slider.api.ClusterDescription; import org.apache.slider.api.ClusterNode; +import org.apache.slider.api.resource.Application; import org.apache.slider.api.types.ComponentInformation; import org.apache.slider.server.appmaster.state.RoleInstance; import org.apache.slider.server.appmaster.web.WebAppApi; @@ -141,18 +141,10 @@ protected void render(Block html) { })); - ClusterDescription desc = appState.getClusterStatus(); - Map options = desc.getRole(name); + Application application = appState.getApplication(); Iterable> tableContent; - - // Generate the pairs of data in the expected form - if (null != options) { - tableContent = Iterables.transform(options.entrySet(), stringStringPairFunc); - } else { - // Or catch that we have no options and provide "empty" - tableContent = Collections.emptySet(); - } - + tableContent = Collections.emptySet(); + // Generate the options used by this role generateRoleDetails(div, "role-options-wrap", "Role Options", tableContent); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java index 2f99b27..2b87699 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java @@ -24,6 +24,7 @@ import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.UL; import org.apache.slider.api.ClusterDescription; import org.apache.slider.api.StatusKeys; +import org.apache.slider.api.resource.Application; import org.apache.slider.api.types.ApplicationLivenessInformation; import org.apache.slider.common.tools.SliderUtils; import org.apache.slider.core.registry.docstore.ExportEntry; @@ -71,8 +72,7 @@ protected void render(Block html) { // An extra method to make testing easier since you can't make an instance of Block @VisibleForTesting protected void doIndex(Hamlet html, String providerName) { - ClusterDescription clusterStatus = appState.getClusterStatus(); - String name = clusterStatus.name; + String name = appState.getApplicationName(); if (name != null && (name.startsWith(" ") || name.endsWith(" "))) { name = "'" + name + "'"; } @@ -96,23 +96,23 @@ protected void doIndex(Hamlet html, String providerName) { ._(); table1.tr() .td("Create time: ") - .td(getInfoAvoidingNulls(StatusKeys.INFO_CREATE_TIME_HUMAN)) + .td("N/A") ._(); table1.tr() .td("Running since: ") - .td(getInfoAvoidingNulls(StatusKeys.INFO_LIVE_TIME_HUMAN)) + .td("N/A") ._(); table1.tr() .td("Time last flexed: ") - .td(getInfoAvoidingNulls(StatusKeys.INFO_FLEX_TIME_HUMAN)) + .td("N/A") ._(); table1.tr() .td("Application storage path: ") - .td(clusterStatus.dataPath) + .td("N/A") ._(); table1.tr() .td("Application configuration path: ") - .td(clusterStatus.originConfigurationPath) + .td("N/A") ._(); table1._(); div._(); @@ -218,7 +218,7 @@ protected void doIndex(Hamlet html, String providerName) { DIV provider_info = html.div("provider_info"); provider_info.h3(providerName + " information"); UL ul = html.ul(); - addProviderServiceOptions(providerService, ul, clusterStatus); + //TODO render app/cluster status ul._(); provider_info._(); @@ -253,11 +253,6 @@ private String getProviderName() { return providerService.getHumanName(); } - private String getInfoAvoidingNulls(String key) { - String createTime = appState.getClusterStatus().getInfo(key); - - return null == createTime ? "N/A" : createTime; - } protected void addProviderServiceOptions(ProviderService provider, UL ul, ClusterDescription clusterStatus) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/YarnApplicationProbe.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/YarnApplicationProbe.java deleted file mode 100644 index 92df048..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/YarnApplicationProbe.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.servicemonitor; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.yarn.api.records.ApplicationReport; -import org.apache.slider.client.SliderYarnClientImpl; -import org.apache.slider.core.exceptions.UnknownApplicationInstanceException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.List; - -/** - * Probe for YARN application - */ -public class YarnApplicationProbe extends Probe { - protected static final Logger log = LoggerFactory.getLogger( - YarnApplicationProbe.class); - - /** - * Yarn client service - */ - private SliderYarnClientImpl yarnClient; - private final String clustername; - private final String username; - - public YarnApplicationProbe(String clustername, - SliderYarnClientImpl yarnClient, - String name, - Configuration conf, String username) - throws IOException { - super("Port probe " + name + " " + clustername, - conf); - this.clustername = clustername; - this.yarnClient = yarnClient; - this.username = username; - } - - - @Override - public void init() throws IOException { - - log.info("Checking " + clustername ); - } - - /** - * Try to connect to the (host,port); a failure to connect within - * the specified timeout is a failure - * @param livePing is the ping live: true for live; false for boot time - * @return the outcome - */ - @Override - public ProbeStatus ping(boolean livePing) { - ProbeStatus status = new ProbeStatus(); - try { - List instances = yarnClient - .listDeployedInstances(username, null, clustername); - ApplicationReport instance = yarnClient - .findClusterInInstanceList(instances, clustername); - if (null == instance) { - throw UnknownApplicationInstanceException.unknownInstance(clustername); - } - status.succeed(this); - } catch (Exception e) { - status.fail(this, e); - } - return status; - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java index 254bf27..6defa2b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java @@ -210,8 +210,6 @@ public String putService( /** * Add a service under a path for the current user - * @param serviceClass service class to use under ~user - * @param serviceName name of the service * @param record service record * @param deleteTreeFirst perform recursive delete of the path first * @return the path the service was created at diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterMessages.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterMessages.proto index b8bdc59..bfcab23 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterMessages.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterMessages.proto @@ -80,22 +80,14 @@ message UpgradeContainersRequestProto { message UpgradeContainersResponseProto { } -/** - * flex the cluster - */ -message FlexClusterRequestProto { - required string clusterSpec = 1; +message FlexComponentRequestProto { + optional string name = 1; + optional int32 numberOfContainers = 2; } - -/** - * flex the cluster - */ -message FlexClusterResponseProto { - required bool response = 1; +message FlexComponentResponseProto { } - /** * void request */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterProtocol.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterProtocol.proto index d68da2b..8a0faf9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterProtocol.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterProtocol.proto @@ -61,24 +61,14 @@ service SliderClusterProtocolPB { rpc upgradeContainers(UpgradeContainersRequestProto) returns(UpgradeContainersResponseProto); - /** - * Flex the cluster. - */ - rpc flexCluster(FlexClusterRequestProto) - returns(FlexClusterResponseProto); + rpc flexComponent(FlexComponentRequestProto) returns (FlexComponentResponseProto); /** * Get the current cluster status */ rpc getJSONClusterStatus(GetJSONClusterStatusRequestProto) returns(GetJSONClusterStatusResponseProto); - - /** - * Get the instance definition - */ - rpc getInstanceDefinition(GetInstanceDefinitionRequestProto) - returns(GetInstanceDefinitionResponseProto); - + /** * List all running nodes in a role */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/slider.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/slider.xml index a2517d5..96bfe0f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/slider.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/slider.xml @@ -24,10 +24,6 @@ true - slider.provider.agent - org.apache.slider.providers.agent.AgentProviderFactory - - slider.provider.docker org.apache.slider.providers.docker.DockerProviderFactory diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java deleted file mode 100644 index b955931..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java +++ /dev/null @@ -1,157 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.core.launch; - -import java.lang.reflect.Method; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; - -import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; -import org.apache.hadoop.yarn.api.records.LogAggregationContext; -import org.apache.hadoop.yarn.client.api.YarnClientApplication; -import org.apache.slider.api.ResourceKeys; -import org.apache.slider.client.SliderYarnClientImpl; -import org.apache.slider.common.SliderKeys; -import org.easymock.EasyMock; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -public class TestAppMasterLauncher { - SliderYarnClientImpl mockYarnClient; - YarnClientApplication yarnClientApp; - ApplicationSubmissionContext appSubmissionContext; - Set tags = Collections.emptySet(); - AppMasterLauncher appMasterLauncher = null; - boolean isOldApi = true; - Method rolledLogsIncludeMethod = null; - Method rolledLogsExcludeMethod = null; - - @Before - public void initialize() throws Exception { - mockYarnClient = EasyMock.createNiceMock(SliderYarnClientImpl.class); - yarnClientApp = EasyMock.createNiceMock(YarnClientApplication.class); - appSubmissionContext = EasyMock - .createNiceMock(ApplicationSubmissionContext.class); - EasyMock.expect(yarnClientApp.getApplicationSubmissionContext()) - .andReturn(appSubmissionContext).once(); - EasyMock.expect(mockYarnClient.createApplication()) - .andReturn(yarnClientApp).once(); - - try { - LogAggregationContext.class.getMethod("newInstance", String.class, - String.class, String.class, String.class); - isOldApi = false; - rolledLogsIncludeMethod = LogAggregationContext.class - .getMethod("getRolledLogsIncludePattern"); - rolledLogsExcludeMethod = LogAggregationContext.class - .getMethod("getRolledLogsExcludePattern"); - } catch (Exception e) { - isOldApi = true; - } - } - - /** - * These tests will probably fail when compiled against hadoop 2.7+. Please - * refer to SLIDER-810. It has been purposely not modified so that it fails - * and that someone needs to modify the code in - * {@code AbstractLauncher#extractLogAggregationContext(Map)}. Comments are - * provided in that method as to what needs to be done. - * - * @throws Exception - */ - @Test - public void testExtractLogAggregationContext() throws Exception { - Map options = new HashMap(); - options.put(ResourceKeys.YARN_LOG_INCLUDE_PATTERNS, - " | slider*.txt |agent.out| |"); - options.put(ResourceKeys.YARN_LOG_EXCLUDE_PATTERNS, - "command*.json| agent.log* | "); - - EasyMock.replay(mockYarnClient, appSubmissionContext, yarnClientApp); - appMasterLauncher = new AppMasterLauncher("cl1", SliderKeys.APP_TYPE, null, - null, mockYarnClient, false, null, options, tags, null); - - // Verify the include/exclude patterns - String expectedInclude = "slider*.txt|agent.out"; - String expectedExclude = "command*.json|agent.log*"; - assertPatterns(expectedInclude, expectedExclude); - - EasyMock.verify(mockYarnClient, appSubmissionContext, yarnClientApp); - - } - - @Test - public void testExtractLogAggregationContextEmptyIncludePattern() - throws Exception { - Map options = new HashMap(); - options.put(ResourceKeys.YARN_LOG_INCLUDE_PATTERNS, " "); - options.put(ResourceKeys.YARN_LOG_EXCLUDE_PATTERNS, - "command*.json| agent.log* | "); - - EasyMock.replay(mockYarnClient, appSubmissionContext, yarnClientApp); - appMasterLauncher = new AppMasterLauncher("cl1", SliderKeys.APP_TYPE, null, - null, mockYarnClient, false, null, options, tags, null); - - // Verify the include/exclude patterns - String expectedInclude = isOldApi ? "" : ".*"; - String expectedExclude = "command*.json|agent.log*"; - assertPatterns(expectedInclude, expectedExclude); - - EasyMock.verify(mockYarnClient, appSubmissionContext, yarnClientApp); - } - - @Test - public void testExtractLogAggregationContextEmptyIncludeAndExcludePattern() - throws Exception { - Map options = new HashMap(); - options.put(ResourceKeys.YARN_LOG_INCLUDE_PATTERNS, ""); - options.put(ResourceKeys.YARN_LOG_EXCLUDE_PATTERNS, " "); - - EasyMock.replay(mockYarnClient, appSubmissionContext, yarnClientApp); - appMasterLauncher = new AppMasterLauncher("cl1", SliderKeys.APP_TYPE, null, - null, mockYarnClient, false, null, options, tags, null); - - // Verify the include/exclude patterns - String expectedInclude = isOldApi ? "" : ".*"; - String expectedExclude = ""; - assertPatterns(expectedInclude, expectedExclude); - - EasyMock.verify(mockYarnClient, appSubmissionContext, yarnClientApp); - } - - private void assertPatterns(String expectedIncludePattern, - String expectedExcludePattern) throws Exception { - if (isOldApi) { - Assert.assertEquals(expectedIncludePattern, - appMasterLauncher.logAggregationContext.getIncludePattern()); - Assert.assertEquals(expectedExcludePattern, - appMasterLauncher.logAggregationContext.getExcludePattern()); - } else { - Assert.assertEquals(expectedIncludePattern, - (String) rolledLogsIncludeMethod - .invoke(appMasterLauncher.logAggregationContext)); - Assert.assertEquals(expectedExcludePattern, - (String) rolledLogsExcludeMethod - .invoke(appMasterLauncher.logAggregationContext)); - } - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncherWithAmReset.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncherWithAmReset.java deleted file mode 100644 index a8f6b26..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncherWithAmReset.java +++ /dev/null @@ -1,92 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.core.launch; - -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; - -import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; -import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; -import org.apache.hadoop.yarn.client.api.YarnClientApplication; -import org.apache.hadoop.yarn.util.Records; -import org.apache.slider.api.ResourceKeys; -import org.apache.slider.client.SliderYarnClientImpl; -import org.apache.slider.common.SliderKeys; -import org.easymock.EasyMock; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -public class TestAppMasterLauncherWithAmReset { - SliderYarnClientImpl mockYarnClient; - YarnClientApplication yarnClientApp; - ApplicationSubmissionContext appSubmissionContext; - GetNewApplicationResponse newApp; - Set tags = Collections.emptySet(); - AppMasterLauncher appMasterLauncher = null; - boolean isOldApi = true; - - @Before - public void initialize() throws Exception { - mockYarnClient = EasyMock.createNiceMock(SliderYarnClientImpl.class); - yarnClientApp = EasyMock.createNiceMock(YarnClientApplication.class); - newApp = EasyMock.createNiceMock(GetNewApplicationResponse.class); - EasyMock.expect(mockYarnClient.createApplication()) - .andReturn(new YarnClientApplication(newApp, - Records.newRecord(ApplicationSubmissionContext.class))); - } - - @Test - public void testExtractYarnResourceManagerAmRetryCountWindowMs() throws - Exception { - Map options = new HashMap(); - final String expectedInterval = Integer.toString (120000); - options.put(ResourceKeys.YARN_RESOURCEMANAGER_AM_RETRY_COUNT_WINDOW_MS, - expectedInterval); - EasyMock.replay(mockYarnClient, yarnClientApp); - - appMasterLauncher = new AppMasterLauncher("am1", SliderKeys.APP_TYPE, null, - null, mockYarnClient, false, null, options, tags, null); - - ApplicationSubmissionContext ctx = appMasterLauncher.application - .getApplicationSubmissionContext(); - String retryIntervalWindow = Long.toString(ctx - .getAttemptFailuresValidityInterval()); - Assert.assertEquals(expectedInterval, retryIntervalWindow); - } - - @Test - public void testExtractYarnResourceManagerAmRetryCountWindowMsDefaultValue() - throws Exception { - Map options = new HashMap(); - EasyMock.replay(mockYarnClient, yarnClientApp); - - appMasterLauncher = new AppMasterLauncher("am1", SliderKeys.APP_TYPE, null, - null, mockYarnClient, false, null, options, tags, null); - - ApplicationSubmissionContext ctx = appMasterLauncher.application - .getApplicationSubmissionContext(); - long retryIntervalWindow = ctx.getAttemptFailuresValidityInterval(); - Assert.assertEquals(ResourceKeys.DEFAULT_AM_RETRY_COUNT_WINDOW_MS, - retryIntervalWindow); - } - -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/TestServiceRecordAttributes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/TestServiceRecordAttributes.java deleted file mode 100644 index a1986cd..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/TestServiceRecordAttributes.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.server.appmaster; - -import org.apache.hadoop.registry.client.types.ServiceRecord; -import org.apache.slider.common.SliderKeys; -import org.apache.slider.core.conf.MapOperations; -import org.junit.Assert; -import org.junit.Test; - -import java.util.HashMap; -import java.util.Map; - -/** - * - */ -public class TestServiceRecordAttributes extends Assert { - - @Test - public void testAppConfigProvidedServiceRecordAttributes() throws Exception { - Map options = new HashMap<>(); - options.put("slider.some.arbitrary.option", "arbitrary value"); - options.put("service.record.attribute.one_attribute", "one_attribute_value"); - options.put("service.record.attribute.second_attribute", "second_attribute_value"); - MapOperations serviceProps = new MapOperations(SliderKeys.COMPONENT_AM, options); - options = new HashMap<>(); - options.put("some.component.attribute", "component_attribute_value"); - options.put("service.record.attribute.component_attribute", "component_attribute_value"); - MapOperations compProps = new MapOperations("TEST_COMP", options); - - SliderAppMaster appMaster = new SliderAppMaster(); - - ServiceRecord appServiceRecord = new ServiceRecord(); - - appMaster.setProvidedServiceRecordAttributes(serviceProps, appServiceRecord); - - assertNull("property should not be attribute", - appServiceRecord.get("slider.some.arbitrary.option")); - assertEquals("wrong value", "one_attribute_value", - appServiceRecord.get("one_attribute")); - assertEquals("wrong value", "second_attribute_value", - appServiceRecord.get("second_attribute")); - - ServiceRecord compServiceRecord = new ServiceRecord(); - - appMaster.setProvidedServiceRecordAttributes(compProps, compServiceRecord); - - assertNull("should not be attribute", - compServiceRecord.get("some.component.attribute")); - assertEquals("wrong value", "component_attribute_value", - compServiceRecord.get("component_attribute")); - - } -}