diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
index bf46d15a650..09140aee1bd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
@@ -46,6 +46,7 @@
import org.apache.hadoop.yarn.client.api.AppAdminClient;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.client.api.YarnClientApplication;
+import org.apache.hadoop.yarn.client.cli.ApplicationCLI;
import org.apache.hadoop.yarn.client.util.YarnClientUtils;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -693,16 +694,22 @@ protected Path addJarResource(String serviceName,
ProviderUtils
.addProviderJar(localResources, ServiceMaster.class, SERVICE_CORE_JAR, fs,
libPath, "lib", false);
- Path dependencyLibTarGzip = fs.getDependencyTarGzip();
+ Path dependencyLibTarGzip = fs.getDependencyTarGzip(false);
if (fs.isFile(dependencyLibTarGzip)) {
- LOG.debug("Loading lib tar from " + fs.getFileSystem().getScheme() + ":/"
- + dependencyLibTarGzip);
+ LOG.info("Loading lib tar from " + dependencyLibTarGzip);
fs.submitTarGzipAndUpdate(localResources);
} else {
+ if (dependencyLibTarGzip != null) {
+ LOG.warn("Property {} has a value {}, but is not a valid file",
+ YarnServiceConf.DEPENDENCY_TARBALL_PATH, dependencyLibTarGzip);
+ }
String[] libs = ServiceUtils.getLibDirs();
- LOG.info("Uploading all dependency jars to HDFS. For faster submission of" +
- " apps, pre-upload dependency jars to HDFS "
- + "using command: yarn app -enableFastLaunch");
+ LOG.info("Uploading all dependency jars to HDFS. For faster submission of"
+ + " apps, set config property {} to the dependency tarball location."
+ + " Dependency tarball can be uploaded to any HDFS path directly"
+ + " or by using command: yarn app -{} []",
+ YarnServiceConf.DEPENDENCY_TARBALL_PATH,
+ ApplicationCLI.ENABLE_FAST_LAUNCH);
for (String libDirProp : libs) {
ProviderUtils.addAllDependencyJars(localResources, fs, libPath, "lib",
libDirProp);
@@ -965,16 +972,18 @@ public YarnClient getYarnClient() {
return this.yarnClient;
}
- public int enableFastLaunch() throws IOException, YarnException {
- return actionDependency(true);
+ public int enableFastLaunch(String destinationFolder)
+ throws IOException, YarnException {
+ return actionDependency(destinationFolder, true);
}
- public int actionDependency(boolean overwrite)
+ public int actionDependency(String destinationFolder, boolean overwrite)
throws IOException, YarnException {
String currentUser = RegistryUtils.currentUser();
LOG.info("Running command as user {}", currentUser);
- Path dependencyLibTarGzip = fs.getDependencyTarGzip();
+ Path dependencyLibTarGzip = fs.getDependencyTarGzip(true,
+ destinationFolder);
// Check if dependency has already been uploaded, in which case log
// appropriately and exit success (unless overwrite has been requested)
@@ -996,6 +1005,8 @@ public int actionDependency(boolean overwrite)
LOG.info("Version Info: " + VersionInfo.getBuildVersion());
fs.copyLocalFileToHdfs(tempLibTarGzipFile, dependencyLibTarGzip,
new FsPermission(YarnServiceConstants.DEPENDENCY_DIR_PERMISSIONS));
+ LOG.info("Set config property {} to : {}",
+ YarnServiceConf.DEPENDENCY_TARBALL_PATH, dependencyLibTarGzip);
return EXIT_SUCCESS;
} else {
return EXIT_FALSE;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
index 22926e23eb4..71c154c77bd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
@@ -94,6 +94,12 @@
public static final int DEFAULT_CONTAINER_RECOVERY_TIMEOUT_MS = 120000;
/**
+ * Where the dependency tarball is located
+ */
+ public static final String DEPENDENCY_TARBALL_PATH = YARN_SERVICE_PREFIX
+ + "framework.path";
+
+ /**
* Get long value for the property. First get from the userConf, if not
* present, get from systemConf.
*
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderService.java
index 9f29c8be31b..7f29d1f5c1b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderService.java
@@ -38,9 +38,9 @@ public void processArtifact(AbstractLauncher launcher,
Path artifact = new Path(instance.getCompSpec().getArtifact().getId());
if (!fileSystem.isFile(artifact)) {
throw new IOException(
- "Package doesn't exist as a resource: " + artifact.toString());
+ "Package doesn't exist as a resource: " + artifact);
}
- log.info("Adding resource {}", artifact.toString());
+ log.info("Adding resource {}", artifact);
LocalResourceType type = LocalResourceType.ARCHIVE;
LocalResource packageResource = fileSystem.createAmResource(artifact, type);
launcher.addLocalResource(APP_LIB_DIR, packageResource);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java
index 6b9d4d51296..71652e1ee5c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java
@@ -295,6 +295,9 @@ public void verifyFileExists(Path path) throws IOException {
* reasons including if file check throws IOException
*/
public boolean isFile(Path path) {
+ if (path == null) {
+ return false;
+ }
boolean isFile = false;
try {
FileStatus status = fileSystem.getFileStatus(path);
@@ -321,26 +324,55 @@ public Path getBaseApplicationPath() {
}
/**
- * Get slider dependency parent dir in HDFS
+ * Get service dependency parent dir in HDFS. EnableFastLaunch uploads the
+ * dependency tarball to this folder.
+ *
+ * @param destinationFolder
+ * optional dir path which when not-null should be used
+ * @return the parent dir path of service-dep.tar.gz in HDFS
+ */
+ public String getDependencyPath(String destinationFolder) {
+ return destinationFolder != null ? destinationFolder
+ : String.format(YarnServiceConstants.DEPENDENCY_DIR,
+ VersionInfo.getVersion());
+ }
+
+ /**
+ * Get service dependency absolute filepath in HDFS
*
- * @return the parent dir path of slider.tar.gz in HDFS
+ * @param forUpload
+ * true if used by enableFastLaunch, false for lookup of the absolute
+ * path
+ * @return the absolute path to service dependency tarball in HDFS
*/
- public Path getDependencyPath() {
- String parentDir = YarnServiceConstants.DEPENDENCY_DIR;
- return new Path(String.format(parentDir, VersionInfo.getVersion()));
+ public Path getDependencyTarGzip(boolean forUpload) {
+ return getDependencyTarGzip(forUpload, null);
}
/**
- * Get slider.tar.gz absolute filepath in HDFS
+ * Get service dependency absolute filepath in HDFS
*
- * @return the absolute path to slider.tar.gz in HDFS
+ * @param forUpload
+ * true if used by enableFastLaunch, false for lookup of the absolute
+ * path
+ * @param destinationFolder
+ * optional dir path which when not-null should be used
+ * @return the absolute path to service dependency tarball in HDFS
*/
- public Path getDependencyTarGzip() {
- Path dependencyLibAmPath = getDependencyPath();
- Path dependencyLibTarGzip = new Path(
- dependencyLibAmPath.toUri().toString(),
- YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_NAME
- + YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_EXT);
+ public Path getDependencyTarGzip(boolean forUpload,
+ String destinationFolder) {
+ Path dependencyLibTarGzip = null;
+ if (forUpload) {
+ dependencyLibTarGzip = new Path(getDependencyPath(destinationFolder),
+ YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_NAME
+ + YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_EXT);
+ } else {
+ String configuredDependencyTarballPath = configuration
+ .get(YarnServiceConf.DEPENDENCY_TARBALL_PATH);
+ if (configuredDependencyTarballPath != null) {
+ dependencyLibTarGzip = new Path(configuredDependencyTarballPath);
+ }
+ }
return dependencyLibTarGzip;
}
@@ -452,7 +484,7 @@ public LocalResource submitFile(File localFile, Path tempPath, String subdir, St
public void submitTarGzipAndUpdate(
Map providerResources) throws IOException,
BadClusterStateException {
- Path dependencyLibTarGzip = getDependencyTarGzip();
+ Path dependencyLibTarGzip = getDependencyTarGzip(false);
LocalResource lc = createAmResource(dependencyLibTarGzip,
LocalResourceType.ARCHIVE);
providerResources.put(YarnServiceConstants.DEPENDENCY_LOCALIZED_DIR_LINK, lc);
@@ -467,8 +499,7 @@ public void copyLocalFileToHdfs(File localPath,
fileSystem.getConf().set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY,
"000");
fileSystem.mkdirs(destPath.getParent(), fp);
- log.info("Copying file {} to {}", localPath.toURI(),
- fileSystem.getScheme() + ":/" + destPath.toUri());
+ log.info("Copying file {} to {}", localPath.toURI(), destPath);
fileSystem.copyFromLocalFile(false, true, new Path(localPath.getPath()),
destPath);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceUtils.java
index 915b8361343..5637720fd7a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceUtils.java
@@ -472,7 +472,8 @@ public static ClasspathConstructor buildClasspath(String sliderConfDir,
classpath.addClassDirectory(sliderConfDir);
}
classpath.addLibDir(libdir);
- if (sliderFileSystem.isFile(sliderFileSystem.getDependencyTarGzip())) {
+ if (sliderFileSystem
+ .isFile(sliderFileSystem.getDependencyTarGzip(false))) {
classpath.addLibDir(YarnServiceConstants.DEPENDENCY_LOCALIZED_DIR_LINK);
}
classpath.addRemoteClasspathEnvVar();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AppAdminClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AppAdminClient.java
index 6aba91a5c2d..55be13b10b7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AppAdminClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AppAdminClient.java
@@ -196,14 +196,18 @@ public abstract int actionFlex(String appName, Map
* faster since the dependencies do not have to be uploaded on each launch.
*
*
+ * @param destinationFolder
+ * an optional HDFS folder where dependency tarball will be uploaded
* @return exit code
- * @throws IOException IOException
- * @throws YarnException exception in client or server
+ * @throws IOException
+ * IOException
+ * @throws YarnException
+ * exception in client or server
*/
@Public
@Unstable
- public abstract int enableFastLaunch() throws IOException,
- YarnException;
+ public abstract int enableFastLaunch(String destinationFolder)
+ throws IOException, YarnException;
/**
*
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
index c751f796c64..471b4d6d0ea 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
@@ -232,9 +232,10 @@ public int run(String[] args) throws Exception {
"the number of components/containers running for an application / " +
"long-running service. Supports absolute or relative changes, such " +
"as +1, 2, or -3.");
- opts.addOption(ENABLE_FAST_LAUNCH, false, "Uploads AM dependencies " +
- "to HDFS to make future launches faster. Supports -appTypes option" +
- " to specify which client implementation to use.");
+ opts.addOption(ENABLE_FAST_LAUNCH, true, "Uploads AM dependencies " +
+ "to HDFS to make future launches faster. Supports -appTypes option " +
+ "to specify which client implementation to use. Optionally a " +
+ "destination folder for the tarball can be specified.");
opts.getOption(LAUNCH_CMD).setArgName("Application Name> Uploads AM dependencies to HDFS");
pw.println(" to make future launches faster.");
pw.println(" Supports -appTypes option to");
pw.println(" specify which client");
pw.println(" implementation to use.");
+ pw.println(" Optionally a destination folder");
+ pw.println(" for the tarball can be");
+ pw.println(" specified.");
pw.println(" -flex Changes number of running");
pw.println(" containers for a component of an");
pw.println(" application / long-running");