diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java index b4f6a2e..5a4de0e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java @@ -50,6 +50,7 @@ import javax.ws.rs.core.Response; import javax.ws.rs.core.Response.Status; import java.io.IOException; +import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -251,10 +252,12 @@ public Response updateComponent(@PathParam("app_name") String appName, .getNumberOfContainers()).build(); } try { - long original = SLIDER_CLIENT.flex(appName, component); - return Response.ok().entity( - "Updating " + componentName + " size from " + original + " to " - + component.getNumberOfContainers()).build(); + Map original = SLIDER_CLIENT.flex(appName, Collections + .singletonMap(component.getName(), + component.getNumberOfContainers())); + return Response.ok().entity("Updating " + componentName + " size from " + + original.get(componentName) + " to " + + component.getNumberOfContainers()).build(); } catch (YarnException | IOException e) { ApplicationStatus status = new ApplicationStatus(); status.setDiagnostics(e.getMessage()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml index ee77ecb..f02f7bb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml @@ -322,16 +322,38 @@ easymock 3.1 test + + + org.objenesis + objenesis + + org.powermock powermock-api-easymock - 1.5 + 1.6.5 test + org.powermock + powermock-module-junit4 + 1.6.5 + + + org.javassist + javassist + + + org.objenesis + objenesis + + + + + javax.servlet.jsp jsp-api runtime @@ -354,6 +376,13 @@ swagger-annotations 1.5.4 + + + org.apache.hadoop + hadoop-minicluster + test + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/InternalKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/InternalKeys.java index fcaaf0e..f690f5a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/InternalKeys.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/InternalKeys.java @@ -126,9 +126,9 @@ String CHAOS_MONKEY_INTERVAL_MINUTES = CHAOS_MONKEY_INTERVAL + ".minutes"; String CHAOS_MONKEY_INTERVAL_SECONDS = CHAOS_MONKEY_INTERVAL + ".seconds"; - int DEFAULT_CHAOS_MONKEY_INTERVAL_DAYS = 0; - int DEFAULT_CHAOS_MONKEY_INTERVAL_HOURS = 0; - int DEFAULT_CHAOS_MONKEY_INTERVAL_MINUTES = 0; + long DEFAULT_CHAOS_MONKEY_INTERVAL_DAYS = 0; + long DEFAULT_CHAOS_MONKEY_INTERVAL_HOURS = 0; + long DEFAULT_CHAOS_MONKEY_INTERVAL_MINUTES = 0; String CHAOS_MONKEY_DELAY = "internal.chaos.monkey.delay"; String CHAOS_MONKEY_DELAY_DAYS = CHAOS_MONKEY_DELAY + ".days"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java index 92890be..6cc02e7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java @@ -72,6 +72,14 @@ The window is calculated in minutes as as (days * 24 *60 + hours* 24 + minutes) String YARN_CORES = "yarn.vcores"; /** + * If normalization is set to false, then if the resource (memory and/or + * vcore) requested by a role is higher than YARN limits, then the resource + * request is not normalized. If this causes failures at the YARN level then + * applications are expecting that to happen. Default value is true. + */ + String YARN_RESOURCE_NORMALIZATION_ENABLED = "yarn.resource.normalization.enabled"; + + /** * Number of disks per instance to ask YARN for * {@value} */ @@ -140,9 +148,9 @@ The window is calculated in minutes as as (days * 24 *60 + hours* 24 + minutes) - int DEFAULT_CONTAINER_FAILURE_WINDOW_DAYS = 0; - int DEFAULT_CONTAINER_FAILURE_WINDOW_HOURS = 6; - int DEFAULT_CONTAINER_FAILURE_WINDOW_MINUTES = 0; + long DEFAULT_CONTAINER_FAILURE_WINDOW_DAYS = 0; + long DEFAULT_CONTAINER_FAILURE_WINDOW_HOURS = 6; + long DEFAULT_CONTAINER_FAILURE_WINDOW_MINUTES = 0; /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java index 7f768b9..448d4ba 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java @@ -53,8 +53,8 @@ YarnException; - Messages.FlexComponentResponseProto flexComponent( - Messages.FlexComponentRequestProto request) throws IOException; + Messages.FlexComponentsResponseProto flexComponents( + Messages.FlexComponentsRequestProto request) throws IOException; /** * Get the current cluster status diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java index c43bd64..7b3b93e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java @@ -116,6 +116,28 @@ public long getPropertyLong(String name, long defaultValue) { return Long.parseLong(value); } + public int getPropertyInt(String name, int defaultValue) { + if (name == null) { + return defaultValue; + } + String value = properties.get(name.trim()); + if (StringUtils.isEmpty(value)) { + return defaultValue; + } + return Integer.parseInt(value); + } + + public boolean getPropertyBool(String name, boolean defaultValue) { + if (name == null) { + return defaultValue; + } + String value = properties.get(name.trim()); + if (StringUtils.isEmpty(value)) { + return defaultValue; + } + return Boolean.parseBoolean(value); + } + public String getProperty(String name, String defaultValue) { if (name == null) { return defaultValue; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/RoleStatistics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/RoleStatistics.java index c926600..25f4d9d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/RoleStatistics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/RoleStatistics.java @@ -35,11 +35,8 @@ public long limitsExceeded = 0L; public long nodeFailed = 0L; public long preempted = 0L; - public long releasing = 0L; public long requested = 0L; public long started = 0L; - public long startFailed = 0L; - public long totalRequested = 0L; /** * Add another statistics instance @@ -56,11 +53,8 @@ public RoleStatistics add(final RoleStatistics that) { limitsExceeded += that.limitsExceeded; nodeFailed += that.nodeFailed; preempted += that.preempted; - releasing += that.releasing; requested += that.requested; started += that.started; - startFailed += that.totalRequested; - totalRequested += that.totalRequested; return this; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java index 8bceddf..8e15683 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -20,12 +20,9 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.io.Files; -import org.apache.commons.io.IOUtils; import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; @@ -67,7 +64,6 @@ import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.util.Times; -import org.apache.slider.api.ClusterNode; import org.apache.slider.api.SliderClusterProtocol; import org.apache.slider.api.proto.Messages; import org.apache.slider.api.resource.Application; @@ -83,22 +79,18 @@ import org.apache.slider.common.params.AbstractClusterBuildingActionArgs; import org.apache.slider.common.params.ActionAMSuicideArgs; import org.apache.slider.common.params.ActionClientArgs; -import org.apache.slider.common.params.ActionCreateArgs; import org.apache.slider.common.params.ActionDependencyArgs; import org.apache.slider.common.params.ActionDiagnosticArgs; -import org.apache.slider.common.params.ActionEchoArgs; import org.apache.slider.common.params.ActionExistsArgs; import org.apache.slider.common.params.ActionFlexArgs; import org.apache.slider.common.params.ActionFreezeArgs; import org.apache.slider.common.params.ActionInstallKeytabArgs; -import org.apache.slider.common.params.ActionInstallPackageArgs; import org.apache.slider.common.params.ActionKDiagArgs; import org.apache.slider.common.params.ActionKeytabArgs; import org.apache.slider.common.params.ActionKillContainerArgs; import org.apache.slider.common.params.ActionListArgs; import org.apache.slider.common.params.ActionLookupArgs; import org.apache.slider.common.params.ActionNodesArgs; -import org.apache.slider.common.params.ActionPackageArgs; import org.apache.slider.common.params.ActionRegistryArgs; import org.apache.slider.common.params.ActionResolveArgs; import org.apache.slider.common.params.ActionResourceArgs; @@ -122,7 +114,6 @@ import org.apache.slider.core.exceptions.SliderException; import org.apache.slider.core.exceptions.UnknownApplicationInstanceException; import org.apache.slider.core.exceptions.UsageException; -import org.apache.slider.core.exceptions.WaitTimeoutException; import org.apache.slider.core.launch.ClasspathConstructor; import org.apache.slider.core.launch.CredentialUtils; import org.apache.slider.core.launch.JavaCommandLineBuilder; @@ -144,7 +135,6 @@ import org.apache.slider.providers.AbstractClientProvider; import org.apache.slider.providers.ProviderUtils; import org.apache.slider.providers.SliderProviderFactory; -import org.apache.slider.providers.agent.AgentKeys; import org.apache.slider.server.appmaster.SliderAppMaster; import org.apache.slider.server.appmaster.rpc.RpcBinder; import org.apache.slider.server.services.utility.AbstractSliderLaunchedService; @@ -160,11 +150,9 @@ import org.slf4j.LoggerFactory; import java.io.ByteArrayOutputStream; -import java.io.Console; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; -import java.io.InputStream; import java.io.InterruptedIOException; import java.io.OutputStreamWriter; import java.io.PrintStream; @@ -177,8 +165,6 @@ import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedList; import java.util.List; import java.util.Locale; import java.util.Map; @@ -226,7 +212,7 @@ private ClientArgs serviceArgs; public ApplicationId applicationId; - + private String deployedClusterName; /** * Cluster operations against the deployed cluster -will be null @@ -334,23 +320,19 @@ public int exec() throws Throwable { exitCode = actionAmSuicide(clusterName, serviceArgs.getActionAMSuicideArgs()); break; - + + case ACTION_BUILD: + exitCode = actionBuild(getApplicationFromArgs(clusterName, + serviceArgs.getActionBuildArgs())); + break; + case ACTION_CLIENT: exitCode = actionClient(serviceArgs.getActionClientArgs()); break; case ACTION_CREATE: - ActionCreateArgs args = serviceArgs.getActionCreateArgs(); - File file = args.getAppDef(); - Path filePath = new Path(file.getAbsolutePath()); - log.info("Loading app definition from: " + filePath); - Application application = - jsonSerDeser.load(FileSystem.getLocal(getConfig()), filePath); - if(args.lifetime > 0) { - application.setLifetime(args.lifetime); - } - application.setName(clusterName); - actionCreate(application); + actionCreate(getApplicationFromArgs(clusterName, + serviceArgs.getActionCreateArgs())); break; case ACTION_DEPENDENCY: @@ -395,10 +377,6 @@ public int exec() throws Throwable { exitCode = actionInstallKeytab(serviceArgs.getActionInstallKeytabArgs()); break; - case ACTION_INSTALL_PACKAGE: - exitCode = actionInstallPkg(serviceArgs.getActionInstallPackageArgs()); - break; - case ACTION_KEYTAB: exitCode = actionKeytab(serviceArgs.getActionKeytabArgs()); break; @@ -415,10 +393,6 @@ public int exec() throws Throwable { exitCode = actionNodes("", serviceArgs.getActionNodesArgs()); break; - case ACTION_PACKAGE: - exitCode = actionPackage(serviceArgs.getActionPackageArgs()); - break; - case ACTION_REGISTRY: exitCode = actionRegistry(serviceArgs.getActionRegistryArgs()); break; @@ -605,15 +579,16 @@ protected ZKIntegration getZkClient(String clusterName, String user) throws Yarn * force=true by default. */ @Override - public void actionDestroy(String appName) + public int actionDestroy(String appName) throws YarnException, IOException { validateClusterName(appName); + verifyNoLiveApp(appName, "Destroy"); Path appDir = sliderFileSystem.buildClusterDirPath(appName); FileSystem fs = sliderFileSystem.getFileSystem(); if (fs.exists(appDir)) { if (fs.delete(appDir, true)) { log.info("Successfully deleted application + " + appName); - return; + return EXIT_SUCCESS; } else { String message = "Failed to delete application + " + appName + " at: " + appDir; @@ -627,7 +602,29 @@ public void actionDestroy(String appName) log.warn(message); throw new YarnException(message); } - //TODO clean registry + + //TODO clean registry? + String registryPath = SliderRegistryUtils.registryPathForInstance( + appName); + try { + getRegistryOperations().delete(registryPath, true); + } catch (IOException e) { + log.warn("Error deleting registry entry {}: {} ", registryPath, e, e); + } catch (SliderException e) { + log.warn("Error binding to registry {} ", e, e); + } + + // detect any race leading to cluster creation during the check/destroy process + // and report a problem. + if (fs.exists(appDir)) { + throw new SliderException(EXIT_APPLICATION_IN_USE, + appName + ": " + + E_DESTROY_CREATE_RACE_CONDITION + + " : " + + appDir + " still exists"); + } + log.info("Destroyed cluster {}", appName); + return EXIT_SUCCESS; } @@ -648,6 +645,26 @@ public AbstractClientProvider createClientProvider(String provider) return factory.createClientProvider(); } + private Application getApplicationFromArgs(String clusterName, + AbstractClusterBuildingActionArgs args) throws IOException { + File file = args.getAppDef(); + Path filePath = new Path(file.getAbsolutePath()); + log.info("Loading app definition from: " + filePath); + Application application = + jsonSerDeser.load(FileSystem.getLocal(getConfig()), filePath); + if(args.lifetime > 0) { + application.setLifetime(args.lifetime); + } + application.setName(clusterName); + return application; + } + + public int actionBuild(Application application) throws YarnException, + IOException { + Path appDir = checkAppNotExistOnHdfs(application); + persistApp(appDir, application); + return EXIT_SUCCESS; + } public ApplicationId actionCreate(Application application) throws IOException, YarnException { @@ -684,8 +701,7 @@ private ApplicationId submitApp(Application app) } submissionContext.setMaxAppAttempts(conf.getInt(KEY_AM_RESTART_LIMIT, 2)); - Map localResources = - new HashMap(); + Map localResources = new HashMap<>(); // copy local slideram-log4j.properties to hdfs and add to localResources boolean hasSliderAMLog4j = @@ -724,10 +740,16 @@ private ApplicationId submitApp(Application app) amLaunchContext.setLocalResources(localResources); addCredentialsIfSecure(conf, amLaunchContext); submissionContext.setAMContainerSpec(amLaunchContext); - yarnClient.submitApplication(submissionContext); + submitApplication(submissionContext); return submissionContext.getApplicationId(); } + @VisibleForTesting + public ApplicationId submitApplication(ApplicationSubmissionContext context) + throws IOException, YarnException { + return yarnClient.submitApplication(context); + } + private void printLocalResources(Map map) { log.info("Added LocalResource for localization: "); StringBuilder builder = new StringBuilder(); @@ -800,7 +822,7 @@ private String buildCommandLine(String appName, Configuration conf, private Map addAMEnv(Configuration conf, Path tempPath) throws IOException { - Map env = new HashMap(); + Map env = new HashMap<>(); ClasspathConstructor classpath = buildClasspath(SliderKeys.SUBMITTED_CONF_DIR, "lib", sliderFileSystem, getUsingMiniMRCluster()); @@ -929,69 +951,6 @@ public int actionUpgrade(String clustername, ActionUpgradeArgs upgradeArgs) return 0; } -// protected static void checkForCredentials(Configuration conf, -// ConfTree tree, String clusterName) throws IOException { -// if (tree.credentials == null || tree.credentials.isEmpty()) { -// log.info("No credentials requested"); -// return; -// } -// -// Console console = System.console(); -// for (Entry> cred : tree.credentials.entrySet()) { -// String provider = cred.getKey() -// .replaceAll(Pattern.quote("${CLUSTER_NAME}"), clusterName) -// .replaceAll(Pattern.quote("${CLUSTER}"), clusterName); -// List aliases = cred.getValue(); -// if (aliases == null || aliases.isEmpty()) { -// continue; -// } -// Configuration c = new Configuration(conf); -// c.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, provider); -// CredentialProvider credentialProvider = CredentialProviderFactory.getProviders(c).get(0); -// Set existingAliases = new HashSet<>(credentialProvider.getAliases()); -// for (String alias : aliases) { -// if (existingAliases.contains(alias.toLowerCase(Locale.ENGLISH))) { -// log.info("Credentials for " + alias + " found in " + provider); -// } else { -// if (console == null) { -// throw new IOException("Unable to input password for " + alias + -// " because System.console() is null; provider " + provider + -// " must be populated manually"); -// } -// char[] pass = readPassword(alias, console); -// credentialProvider.createCredentialEntry(alias, pass); -// credentialProvider.flush(); -// Arrays.fill(pass, ' '); -// } -// } -// } -// } - - private static char[] readPassword(String alias, Console console) - throws IOException { - char[] cred = null; - - boolean noMatch; - do { - console.printf("%s %s: \n", PASSWORD_PROMPT, alias); - char[] newPassword1 = console.readPassword(); - console.printf("%s %s again: \n", PASSWORD_PROMPT, alias); - char[] newPassword2 = console.readPassword(); - noMatch = !Arrays.equals(newPassword1, newPassword2); - if (noMatch) { - if (newPassword1 != null) Arrays.fill(newPassword1, ' '); - log.info(String.format("Passwords don't match. Try again.")); - } else { - cred = newPassword1; - } - if (newPassword2 != null) Arrays.fill(newPassword2, ' '); - } while (noMatch); - if (cred == null) - throw new IOException("Could not read credentials for " + alias + - " from stdin"); - return cred; - } - @Override public int actionKeytab(ActionKeytabArgs keytabInfo) throws YarnException, IOException { @@ -1085,36 +1044,6 @@ public int actionInstallKeytab(ActionInstallKeytabArgs installKeytabInfo) } @Override - public int actionInstallPkg(ActionInstallPackageArgs installPkgInfo) throws - YarnException, - IOException { - log.warn("The " + ACTION_INSTALL_PACKAGE - + " option has been deprecated. Please use '" - + ACTION_PACKAGE + " " + ClientArgs.ARG_INSTALL + "'."); - if (StringUtils.isEmpty(installPkgInfo.name)) { - throw new BadCommandArgumentsException( - E_INVALID_APPLICATION_TYPE_NAME + "\n" - + CommonArgs.usage(serviceArgs, ACTION_INSTALL_PACKAGE)); - } - Path srcFile = extractPackagePath(installPkgInfo.packageURI); - - // Do not provide new options to install-package command as it is in - // deprecated mode. So version is kept null here. Use package --install. - Path pkgPath = sliderFileSystem.buildPackageDirPath(installPkgInfo.name, - null); - FileSystem sfs = sliderFileSystem.getFileSystem(); - sfs.mkdirs(pkgPath); - - Path fileInFs = new Path(pkgPath, srcFile.getName()); - log.info("Installing package {} at {} and overwrite is {}.", - srcFile, fileInFs, installPkgInfo.replacePkg); - require(!(sfs.exists(fileInFs) && !installPkgInfo.replacePkg), - "Package exists at %s. : %s", fileInFs.toUri(), E_USE_REPLACEPKG_TO_OVERWRITE); - sfs.copyFromLocalFile(false, installPkgInfo.replacePkg, srcFile, fileInFs); - return EXIT_SUCCESS; - } - - @Override public int actionResource(ActionResourceArgs resourceInfo) throws YarnException, IOException { if (resourceInfo.help) { @@ -1287,236 +1216,6 @@ private int doClientInstall(ActionClientArgs clientInfo) return EXIT_SUCCESS; } - - @Override - public int actionPackage(ActionPackageArgs actionPackageInfo) - throws YarnException, IOException { - initializeOutputStream(actionPackageInfo.out); - int exitCode = -1; - if (actionPackageInfo.help) { - exitCode = actionHelp(ACTION_PACKAGE); - } - if (actionPackageInfo.install) { - exitCode = actionPackageInstall(actionPackageInfo); - } - if (actionPackageInfo.delete) { - exitCode = actionPackageDelete(actionPackageInfo); - } - if (actionPackageInfo.list) { - exitCode = actionPackageList(); - } - if (actionPackageInfo.instances) { - exitCode = actionPackageInstances(); - } - finalizeOutputStream(actionPackageInfo.out); - if (exitCode != -1) { - return exitCode; - } - throw new BadCommandArgumentsException( - "Select valid package operation option"); - } - - private void initializeOutputStream(String outFile) - throws IOException { - if (outFile != null) { - clientOutputStream = new PrintStream(outFile, "UTF-8"); - } else { - clientOutputStream = System.out; - } - } - - private void finalizeOutputStream(String outFile) { - if (outFile != null && clientOutputStream != null) { - clientOutputStream.flush(); - clientOutputStream.close(); - } - clientOutputStream = System.out; - } - - private int actionPackageInstances() throws YarnException, IOException { -// Map persistentInstances = sliderFileSystem -// .listPersistentInstances(); -// if (persistentInstances.isEmpty()) { -// log.info("No slider cluster specification available"); -// return EXIT_SUCCESS; -// } -// String pkgPathValue = sliderFileSystem -// .buildPackageDirPath(StringUtils.EMPTY, StringUtils.EMPTY).toUri() -// .getPath(); -// FileSystem fs = sliderFileSystem.getFileSystem(); -// Iterator> instanceItr = persistentInstances -// .entrySet().iterator(); -// log.info("List of applications with its package name and path"); -// println("%-25s %15s %30s %s", "Cluster Name", "Package Name", -// "Package Version", "Application Location"); - //TODO deal with packages -// while(instanceItr.hasNext()) { -// Map.Entry entry = instanceItr.next(); -// String clusterName = entry.getKey(); -// Path clusterPath = entry.getValue(); -// AggregateConf instanceDefinition = loadInstanceDefinitionUnresolved( -// clusterName, clusterPath); -// Path appDefPath = null; -// try { -// appDefPath = new Path( -// getApplicationDefinitionPath(instanceDefinition -// .getAppConfOperations())); -// } catch (BadConfigException e) { -// // Invalid cluster state, so move on to next. No need to log anything -// // as this is just listing of instances. -// continue; -// } -// if (!appDefPath.isUriPathAbsolute()) { -// appDefPath = new Path(fs.getHomeDirectory(), appDefPath); -// } -// String appDefPathStr = appDefPath.toUri().toString(); -// try { -// if (appDefPathStr.contains(pkgPathValue) && fs.isFile(appDefPath)) { -// String packageName = appDefPath.getParent().getName(); -// String packageVersion = StringUtils.EMPTY; -// if (instanceDefinition.isVersioned()) { -// packageVersion = packageName; -// packageName = appDefPath.getParent().getParent().getName(); -// } -// println("%-25s %15s %30s %s", clusterName, packageName, -// packageVersion, appDefPathStr); -// } -// } catch (IOException e) { -// log.debug("{} application definition path {} is not found.", clusterName, appDefPathStr); -// } -// } - return EXIT_SUCCESS; - } - - private int actionPackageList() throws IOException { - Path pkgPath = sliderFileSystem.buildPackageDirPath(StringUtils.EMPTY, - StringUtils.EMPTY); - log.info("Package install path : {}", pkgPath); - FileSystem sfs = sliderFileSystem.getFileSystem(); - if (!sfs.isDirectory(pkgPath)) { - log.info("No package(s) installed"); - return EXIT_SUCCESS; - } - FileStatus[] fileStatus = sfs.listStatus(pkgPath); - boolean hasPackage = false; - StringBuilder sb = new StringBuilder(); - sb.append("List of installed packages:\n"); - for (FileStatus fstat : fileStatus) { - if (fstat.isDirectory()) { - sb.append("\t").append(fstat.getPath().getName()); - sb.append("\n"); - hasPackage = true; - } - } - if (hasPackage) { - println(sb.toString()); - } else { - log.info("No package(s) installed"); - } - return EXIT_SUCCESS; - } - - private void createSummaryMetainfoFile(Path srcFile, Path destFile, - boolean overwrite) throws IOException { - FileSystem srcFs = srcFile.getFileSystem(getConfig()); - try (InputStream inputStreamJson = SliderUtils - .getApplicationResourceInputStream(srcFs, srcFile, "metainfo.json"); - InputStream inputStreamXml = SliderUtils - .getApplicationResourceInputStream(srcFs, srcFile, "metainfo.xml");) { - InputStream inputStream = null; - Path summaryFileInFs = null; - if (inputStreamJson != null) { - inputStream = inputStreamJson; - summaryFileInFs = new Path(destFile.getParent(), destFile.getName() - + ".metainfo.json"); - log.info("Found JSON metainfo file in package"); - } else if (inputStreamXml != null) { - inputStream = inputStreamXml; - summaryFileInFs = new Path(destFile.getParent(), destFile.getName() - + ".metainfo.xml"); - log.info("Found XML metainfo file in package"); - } - if (inputStream != null) { - try (FSDataOutputStream dataOutputStream = sliderFileSystem - .getFileSystem().create(summaryFileInFs, overwrite)) { - log.info("Creating summary metainfo file"); - IOUtils.copy(inputStream, dataOutputStream); - } - } - } - } - - private int actionPackageInstall(ActionPackageArgs actionPackageArgs) - throws YarnException, IOException { - requireArgumentSet(Arguments.ARG_NAME, actionPackageArgs.name); - - Path srcFile = extractPackagePath(actionPackageArgs.packageURI); - - Path pkgPath = sliderFileSystem.buildPackageDirPath(actionPackageArgs.name, - actionPackageArgs.version); - FileSystem fs = sliderFileSystem.getFileSystem(); - if (!fs.exists(pkgPath)) { - fs.mkdirs(pkgPath); - } - - Path fileInFs = new Path(pkgPath, srcFile.getName()); - require(actionPackageArgs.replacePkg || !fs.exists(fileInFs), - E_PACKAGE_EXISTS +" at %s. Use --replacepkg to overwrite.", fileInFs.toUri()); - - log.info("Installing package {} to {} (overwrite set to {})", srcFile, - fileInFs, actionPackageArgs.replacePkg); - fs.copyFromLocalFile(false, actionPackageArgs.replacePkg, srcFile, fileInFs); - createSummaryMetainfoFile(srcFile, fileInFs, actionPackageArgs.replacePkg); - - String destPathWithHomeDir = Path - .getPathWithoutSchemeAndAuthority(fileInFs).toString(); - String destHomeDir = Path.getPathWithoutSchemeAndAuthority( - fs.getHomeDirectory()).toString(); - // a somewhat contrived approach to stripping out the home directory and any trailing - // separator; designed to work on windows and unix - String destPathWithoutHomeDir; - if (destPathWithHomeDir.startsWith(destHomeDir)) { - destPathWithoutHomeDir = destPathWithHomeDir.substring(destHomeDir.length()); - if (destPathWithoutHomeDir.startsWith("/") || destPathWithoutHomeDir.startsWith("\\")) { - destPathWithoutHomeDir = destPathWithoutHomeDir.substring(1); - } - } else { - destPathWithoutHomeDir = destPathWithHomeDir; - } - log.info("Set " + AgentKeys.APP_DEF + " in your app config JSON to {}", - destPathWithoutHomeDir); - - return EXIT_SUCCESS; - } - - private Path extractPackagePath(String packageURI) - throws BadCommandArgumentsException { - require(isSet(packageURI), E_INVALID_APPLICATION_PACKAGE_LOCATION); - File pkgFile = new File(packageURI); - require(pkgFile.isFile(), - E_UNABLE_TO_READ_SUPPLIED_PACKAGE_FILE + ": " + pkgFile.getAbsolutePath()); - return new Path(pkgFile.toURI()); - } - - private int actionPackageDelete(ActionPackageArgs actionPackageArgs) throws - YarnException, IOException { - requireArgumentSet(Arguments.ARG_NAME, actionPackageArgs.name); - - Path pkgPath = sliderFileSystem.buildPackageDirPath(actionPackageArgs.name, - actionPackageArgs.version); - FileSystem fs = sliderFileSystem.getFileSystem(); - require(fs.exists(pkgPath), E_PACKAGE_DOES_NOT_EXIST +": %s ", pkgPath.toUri()); - log.info("Deleting package {} at {}.", actionPackageArgs.name, pkgPath); - - if(fs.delete(pkgPath, true)) { - log.info("Deleted package {} " + actionPackageArgs.name); - return EXIT_SUCCESS; - } else { - log.warn("Package deletion failed."); - return EXIT_NOT_FOUND; - } - } - @Override public int actionUpdate(String clustername, AbstractClusterBuildingActionArgs buildInfo) throws @@ -1594,7 +1293,7 @@ public String updateLifetime(String appName, long lifetime) String PLACEHOLDER_PATTERN = "\\$\\{[^{]+\\}"; Pattern placeholderPattern = Pattern.compile(PLACEHOLDER_PATTERN); Matcher placeholderMatcher = placeholderPattern.matcher(env); - Map placeholderKeyValueMap = new HashMap(); + Map placeholderKeyValueMap = new HashMap<>(); if (placeholderMatcher.find()) { String placeholderKey = placeholderMatcher.group(); String systemKey = placeholderKey @@ -1865,16 +1564,20 @@ public boolean isApplicationActive(ApplicationReport report) { @Override @VisibleForTesting - public void actionFlex(String appName, ActionFlexArgs args) + public int actionFlex(String appName, ActionFlexArgs args) throws YarnException, IOException { - Component component = new Component(); - component.setNumberOfContainers(args.getNumberOfContainers()); - if (StringUtils.isEmpty(args.getComponent())) { - component.setName("DEFAULT"); - } else { - component.setName(args.getComponent()); + Map componentCounts = new HashMap<>(args.getComponentMap() + .size()); + for (Entry entry : args.getComponentMap().entrySet()) { + long numberOfContainers = Long.valueOf(entry.getValue()); + componentCounts.put(entry.getKey(), numberOfContainers); + } + // throw usage exception if no changes proposed + if (componentCounts.size() == 0) { + actionHelp(ACTION_FLEX); } - flex(appName, component); + flex(appName, componentCounts); + return EXIT_SUCCESS; } @Override @@ -1966,19 +1669,6 @@ public int actionKillContainer(String name, return EXIT_SUCCESS; } - @Override - public String actionEcho(String name, ActionEchoArgs args) throws - YarnException, - IOException { - String message = args.message; - if (message == null) { - throw new BadCommandArgumentsException("missing message"); - } - SliderClusterOperations clusterOps = - new SliderClusterOperations(bondToCluster(name)); - return clusterOps.echo(message); - } - /** * Find an instance of an application belonging to the current user. * @param appname application name @@ -2099,7 +1789,7 @@ public int actionVersion() { } @Override - public void actionStop(String appName, ActionFreezeArgs freezeArgs) + public int actionStop(String appName, ActionFreezeArgs freezeArgs) throws YarnException, IOException { validateClusterName(appName); ApplicationReport app = findInstance(appName); @@ -2112,7 +1802,7 @@ public void actionStop(String appName, ActionFreezeArgs freezeArgs) .ordinal()) { log.info("Application {} is in a terminated state {}", appName, app.getYarnApplicationState()); - return; + return EXIT_SUCCESS; } try { @@ -2127,6 +1817,7 @@ public void actionStop(String appName, ActionFreezeArgs freezeArgs) + " gracefully, forcefully kill the app."); yarnClient.killApplication(app.getApplicationId(), freezeArgs.message); } + return EXIT_SUCCESS; } @Override @@ -2143,30 +1834,30 @@ public int actionStart(String appName, ActionThawArgs thaw) return 0; } - public long flex(String appName, Component component) - throws YarnException, IOException { + public Map flex(String appName, Map + componentCounts) throws YarnException, IOException { validateClusterName(appName); Path appDir = sliderFileSystem.buildClusterDirPath(appName); Path appJson = new Path(appDir, appName + ".json"); Application persistedApp = jsonSerDeser.load(sliderFileSystem.getFileSystem(), appJson); - long original = 0; - boolean foundComponent = false; + Map original = new HashMap<>(componentCounts.size()); for (Component persistedComp : persistedApp.getComponents()) { - if (persistedComp.getName().equals(component.getName())) { - original = persistedComp.getNumberOfContainers(); - persistedComp.setNumberOfContainers(component.getNumberOfContainers()); - foundComponent = true; - break; + String name = persistedComp.getName(); + if (componentCounts.containsKey(persistedComp.getName())) { + original.put(name, persistedComp.getNumberOfContainers()); + persistedComp.setNumberOfContainers(componentCounts.get(name)); } } - if (!foundComponent) { - throw new YarnException("Component " + component.getName() - + " does not exist in app definition."); + if (original.size() < componentCounts.size()) { + componentCounts.keySet().removeAll(original.keySet()); + throw new YarnException("Components " + componentCounts.keySet() + + " do not exist in app definition."); } jsonSerDeser .save(sliderFileSystem.getFileSystem(), appJson, persistedApp, true); - log.info("Updated app definition file for component " + component); + log.info("Updated app definition file for components " + componentCounts + .keySet()); ApplicationReport instance = findInstance(appName); if (instance != null) { @@ -2174,11 +1865,14 @@ public long flex(String appName, Component component) SliderClusterProtocol appMaster = connect(instance); SliderClusterOperations clusterOps = new SliderClusterOperations(appMaster); - clusterOps.flex(component); - log.info( - "Application name = " + appName + ", Component name = " + component - .getName() + ", number of containers updated from " + original - + " to " + component.getNumberOfContainers()); + clusterOps.flex(componentCounts); + for (Entry componentCount : componentCounts.entrySet()) { + log.info( + "Application name = " + appName + ", Component name = " + + componentCount.getKey() + ", number of containers updated " + + "from " + original.get(componentCount.getKey()) + " to " + + componentCount.getValue()); + } } else { String message = "Application " + appName + "does not exist in RM. "; throw new YarnException(message); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java index 2bb224b..026cb18 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java @@ -20,26 +20,19 @@ import org.apache.hadoop.registry.client.api.RegistryOperations; import org.apache.hadoop.service.Service; -import org.apache.hadoop.yarn.api.records.ApplicationReport; -import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.slider.api.resource.Application; import org.apache.slider.api.types.NodeInformationList; -import org.apache.slider.api.types.SliderInstanceDescription; import org.apache.slider.common.params.AbstractClusterBuildingActionArgs; import org.apache.slider.common.params.ActionAMSuicideArgs; import org.apache.slider.common.params.ActionClientArgs; import org.apache.slider.common.params.ActionDependencyArgs; -import org.apache.slider.common.params.ActionDestroyArgs; import org.apache.slider.common.params.ActionDiagnosticArgs; -import org.apache.slider.common.params.ActionEchoArgs; import org.apache.slider.common.params.ActionFlexArgs; import org.apache.slider.common.params.ActionFreezeArgs; import org.apache.slider.common.params.ActionInstallKeytabArgs; -import org.apache.slider.common.params.ActionInstallPackageArgs; import org.apache.slider.common.params.ActionKeytabArgs; import org.apache.slider.common.params.ActionNodesArgs; -import org.apache.slider.common.params.ActionPackageArgs; import org.apache.slider.common.params.ActionKillContainerArgs; import org.apache.slider.common.params.ActionListArgs; import org.apache.slider.common.params.ActionRegistryArgs; @@ -53,7 +46,6 @@ import org.apache.slider.providers.AbstractClientProvider; import java.io.IOException; -import java.util.Map; /** * Interface of those method calls in the slider API that are intended @@ -63,8 +55,7 @@ */ public interface SliderClientAPI extends Service { - void actionDestroy(String clustername) throws YarnException, - IOException; + int actionDestroy(String clustername) throws YarnException, IOException; /** * AM to commit an asynchronous suicide @@ -105,17 +96,6 @@ int actionKeytab(ActionKeytabArgs keytabInfo) throws YarnException, IOException; /** - * Upload application package to user home directory - * - * @param installPkgInfo the arguments needed to upload the package - * @throws YarnException Yarn problems - * @throws IOException other problems - * @throws BadCommandArgumentsException bad arguments. - */ - int actionInstallPkg(ActionInstallPackageArgs installPkgInfo) - throws YarnException, IOException; - - /** * Manage file resources leveraged by slider * * @param resourceInfo the arguments needed to manage the resource @@ -138,17 +118,6 @@ int actionClient(ActionClientArgs clientInfo) throws IOException, YarnException; /** - * Managing slider application package - * - * @param pkgInfo the arguments needed to upload, delete or list the package - * @throws YarnException Yarn problems - * @throws IOException other problems - * @throws BadCommandArgumentsException bad arguments. - */ - int actionPackage(ActionPackageArgs pkgInfo) - throws YarnException, IOException; - - /** * Update the cluster specification * * @param clustername cluster name @@ -179,7 +148,8 @@ int actionUpgrade(String clustername, int actionList(String clustername, ActionListArgs args) throws IOException, YarnException; - void actionFlex(String name, ActionFlexArgs args) throws YarnException, IOException; + int actionFlex(String name, ActionFlexArgs args) throws YarnException, + IOException; /** * Test for a cluster existing probe for a cluster of the given name existing @@ -200,17 +170,6 @@ int actionKillContainer(String name, ActionKillContainerArgs args) throws YarnException, IOException; /** - * Echo operation (not currently wired up to command line) - * @param name cluster name - * @param args arguments - * @return the echoed text - * @throws YarnException - * @throws IOException - */ - String actionEcho(String name, ActionEchoArgs args) - throws YarnException, IOException; - - /** * Status operation * * @param clustername cluster name @@ -246,7 +205,7 @@ int actionStatus(String clustername, ActionStatusArgs statusArgs) * @param freezeArgs arguments to the stop * @return EXIT_SUCCESS if the cluster was not running by the end of the operation */ - void actionStop(String clustername, ActionFreezeArgs freezeArgs) + int actionStop(String clustername, ActionFreezeArgs freezeArgs) throws YarnException, IOException; /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java index 3bb2af6..d86a0c9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java @@ -24,6 +24,7 @@ import org.apache.slider.api.SliderClusterProtocol; import org.apache.slider.api.StateValues; import org.apache.slider.api.proto.Messages; +import org.apache.slider.api.proto.Messages.ComponentCountProto; import org.apache.slider.api.resource.Application; import org.apache.slider.api.resource.Component; import org.apache.slider.api.types.ContainerInformation; @@ -44,6 +45,8 @@ import java.util.Arrays; import java.util.Collection; import java.util.List; +import java.util.Map; +import java.util.Map.Entry; import static org.apache.slider.api.types.RestTypeMarshalling.unmarshall; @@ -283,12 +286,17 @@ public int waitForRoleInstanceLive(String role, long timeout) return state; } - public void flex(Component component) throws IOException{ - Messages.FlexComponentRequestProto request = - Messages.FlexComponentRequestProto.newBuilder() - .setNumberOfContainers(component.getNumberOfContainers().intValue()) - .setName(component.getName()).build(); - appMaster.flexComponent(request); + public void flex(Map componentCounts) throws IOException{ + Messages.FlexComponentsRequestProto.Builder builder = + Messages.FlexComponentsRequestProto.newBuilder(); + for (Entry componentCount : componentCounts.entrySet()) { + Messages.ComponentCountProto componentProto = + Messages.ComponentCountProto.newBuilder() + .setName(componentCount.getKey()) + .setNumberOfContainers(componentCount.getValue()).build(); + builder.addComponents(componentProto); + } + appMaster.flexComponents(builder.build()); } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java index b666834..f256f9d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java @@ -105,23 +105,6 @@ */ String DEFAULT_DATA_DIRECTORY_PERMISSIONS = "750"; - /** - * - * Use {@link RegistryConstants#KEY_REGISTRY_ZK_ROOT} - * - */ - @Deprecated - String REGISTRY_PATH = "slider.registry.path"; - - /** - * - * @Deprecated use {@link RegistryConstants#KEY_REGISTRY_ZK_QUORUM} - * - */ - @Deprecated - String REGISTRY_ZK_QUORUM = "slider.zookeeper.quorum"; - - String IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH = "ipc.client.fallback-to-simple-auth-allowed"; String HADOOP_HTTP_FILTER_INITIALIZERS = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java index a2d4e38..4052f49 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java @@ -19,6 +19,13 @@ package org.apache.slider.common.params; import com.beust.jcommander.Parameter; +import com.beust.jcommander.ParametersDelegate; +import com.google.common.annotations.VisibleForTesting; +import org.apache.slider.core.exceptions.BadCommandArgumentsException; + +import java.io.File; +import java.util.List; +import java.util.Map; /** * Abstract Action to build things; shares args across build and @@ -26,6 +33,14 @@ */ public abstract class AbstractClusterBuildingActionArgs extends AbstractActionArgs { + @Parameter(names = {ARG_APPDEF}, + description = "Template application definition file in JSON format.") + public File appDef; + + public File getAppDef() { + return appDef; + } + @Parameter(names = { ARG_QUEUE }, description = "Queue to submit the application") public String queue; @@ -33,4 +48,42 @@ @Parameter(names = { ARG_LIFETIME }, description = "Lifetime of the application from the time of request") public long lifetime; + + @ParametersDelegate + public ComponentArgsDelegate componentDelegate = new ComponentArgsDelegate(); + + @ParametersDelegate + public OptionArgsDelegate optionsDelegate = + new OptionArgsDelegate(); + + + public Map getOptionsMap() throws + BadCommandArgumentsException { + return optionsDelegate.getOptionsMap(); + } + + /** + * Get the role heap mapping (may be empty, but never null) + * @return role heap mapping + * @throws BadCommandArgumentsException parse problem + */ + public Map> getCompOptionMap() throws + BadCommandArgumentsException { + return optionsDelegate.getCompOptionMap(); + } + + @VisibleForTesting + public List getComponentTuples() { + return componentDelegate.getComponentTuples(); + } + + /** + * Get the role mapping (may be empty, but never null) + * @return role mapping + * @throws BadCommandArgumentsException parse problem + */ + public Map getComponentMap() throws + BadCommandArgumentsException { + return componentDelegate.getComponentMap(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionBuildArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionBuildArgs.java new file mode 100644 index 0000000..57e4b02 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionBuildArgs.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.common.params; + +import com.beust.jcommander.Parameters; + +@Parameters(commandNames = {SliderActions.ACTION_BUILD}, + commandDescription = SliderActions.DESCRIBE_ACTION_BUILD) + +public class ActionBuildArgs extends AbstractClusterBuildingActionArgs { + + @Override + public String getActionName() { + return SliderActions.ACTION_BUILD; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionCreateArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionCreateArgs.java index c8cac65..842646e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionCreateArgs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionCreateArgs.java @@ -29,14 +29,6 @@ public class ActionCreateArgs extends AbstractClusterBuildingActionArgs { - @Parameter(names = {ARG_APPDEF}, - description = "Template application definition file in JSON format.") - public File appDef; - - public File getAppDef() { - return appDef; - } - @Override public String getActionName() { return SliderActions.ACTION_CREATE; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionEchoArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionEchoArgs.java deleted file mode 100644 index d05f10b..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionEchoArgs.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.common.params; - -import com.beust.jcommander.Parameter; - -public class ActionEchoArgs extends AbstractActionArgs { - @Override - public String getActionName() { - return SliderActions.ACTION_ECHO; - } - - @Parameter(names = {ARG_MESSAGE}, - description = "message to echo") - public String message; - -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionFlexArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionFlexArgs.java index c565484..419f6b3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionFlexArgs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionFlexArgs.java @@ -18,31 +18,37 @@ package org.apache.slider.common.params; -import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; +import com.beust.jcommander.ParametersDelegate; +import org.apache.slider.core.exceptions.BadCommandArgumentsException; + +import java.util.List; +import java.util.Map; @Parameters(commandNames = {SliderActions.ACTION_FLEX}, commandDescription = SliderActions.DESCRIBE_ACTION_FLEX) public class ActionFlexArgs extends AbstractActionArgs { - @Parameter(names = {ARG_COMPONENT}, - description = "component name") - String componentName; - - @Parameter(names = {ARG_COUNT}, - description = "number of containers>") - long numberOfContainers; - @Override public String getActionName() { return SliderActions.ACTION_FLEX; } - public String getComponent() { - return componentName; + @ParametersDelegate + public ComponentArgsDelegate componentDelegate = new ComponentArgsDelegate(); + + /** + * Get the component mapping (may be empty, but never null) + * @return mapping + * @throws BadCommandArgumentsException parse problem + */ + public Map getComponentMap() throws BadCommandArgumentsException { + return componentDelegate.getComponentMap(); } - public long getNumberOfContainers() { - return numberOfContainers; + + public List getComponentTuples() { + return componentDelegate.getComponentTuples(); } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionInstallPackageArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionInstallPackageArgs.java deleted file mode 100644 index 646e795..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionInstallPackageArgs.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.common.params; - -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; -import com.beust.jcommander.ParametersDelegate; - -@Parameters(commandNames = {SliderActions.ACTION_INSTALL_PACKAGE}, - commandDescription = SliderActions.DESCRIBE_ACTION_INSTALL_PACKAGE) - -public class ActionInstallPackageArgs extends AbstractActionArgs { - - @Override - public String getActionName() { - return SliderActions.ACTION_INSTALL_PACKAGE; - } - - @Parameter(names = {ARG_PACKAGE}, - description = "Path to app package on local disk") - public String packageURI; - - @Parameter(names = {ARG_NAME}, - description = "The type of the package") - public String name; - - @Parameter(names = {ARG_REPLACE_PKG}, description = "Overwrite existing package") - public boolean replacePkg = false; - - /** - * Get the min #of params expected - * @return the min number of params in the {@link #parameters} field - */ - public int getMinParams() { - return 0; - } - - @Override - public int getMaxParams() { - return 1; - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionPackageArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionPackageArgs.java deleted file mode 100644 index 4833934..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionPackageArgs.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.common.params; - -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; - -@Parameters(commandNames = {SliderActions.ACTION_PACKAGE}, - commandDescription = SliderActions.DESCRIBE_ACTION_PACKAGE) - -public class ActionPackageArgs extends AbstractActionArgs { - - @Override - public String getActionName() { - return SliderActions.ACTION_PACKAGE; - } - - @Parameter(names = {ARG_INSTALL}, - description = "Install package in the sub-folder 'package' of the user's Slider base directory") - public boolean install; - - @Parameter(names = {ARG_PKGDELETE}, - description = "Delete package operation") - public boolean delete; - - @Parameter(names = {ARG_PKGLIST}, - description = "List of package(s) installed") - public boolean list; - - @Parameter(names = {ARG_PKGINSTANCES}, - description = "Lists all application instances referring to package") - public boolean instances; - - @Parameter(names = {ARG_PACKAGE}, - description = "Path to app package on local disk") - public String packageURI; - - @Parameter(names = {ARG_NAME}, - description = "Package name") - public String name; - - @Parameter(names = {ARG_VERSION}, description = "Package version") - public String version; - - @Parameter(names = {ARG_REPLACE_PKG}, - description = "Overwrite existing package") - public boolean replacePkg = false; - - @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT}, - description = "Output file for package data") - public String out; - - /** - * Get the min #of params expected - * @return the min number of params in the {@link #parameters} field - */ - public int getMinParams() { - return 0; - } - - @Override - public int getMaxParams() { - return 1; - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionUpgradeArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionUpgradeArgs.java index 18aa1f5..eb87108 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionUpgradeArgs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionUpgradeArgs.java @@ -26,7 +26,7 @@ @Parameters(commandNames = { SliderActions.ACTION_UPGRADE }, commandDescription = SliderActions.DESCRIBE_ACTION_UPGRADE) -public class ActionUpgradeArgs extends AbstractActionArgs { +public class ActionUpgradeArgs extends AbstractClusterBuildingActionArgs { @Override public String getActionName() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AppAndResouceOptionArgsDelegate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AppAndResouceOptionArgsDelegate.java deleted file mode 100644 index f171708..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AppAndResouceOptionArgsDelegate.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.common.params; - -import com.beust.jcommander.Parameter; -import org.apache.slider.core.exceptions.BadCommandArgumentsException; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -/** - * Delegate for application and resource options - */ -public class AppAndResouceOptionArgsDelegate extends AbstractArgsDelegate { - - - /** - * Options key value - */ - @Parameter(names = {ARG_OPTION, ARG_OPTION_SHORT}, arity = 2, - description = ARG_OPTION + " ", - splitter = DontSplitArguments.class) - public List optionTuples = new ArrayList<>(0); - - - /** - * All the app component option triples - */ - @Parameter(names = {ARG_COMP_OPT, ARG_COMP_OPT_SHORT, ARG_ROLEOPT}, arity = 3, - description = "Component option " + ARG_COMP_OPT + - "