diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java index 453619b9729..e77f020ff2d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java @@ -812,13 +812,13 @@ private String buildCommandLine(Service app, Configuration conf, protected Path addJarResource(String serviceName, Map localResources) - throws IOException, SliderException { + throws IOException, YarnException { Path libPath = fs.buildClusterDirPath(serviceName); ProviderUtils .addProviderJar(localResources, ServiceMaster.class, SERVICE_CORE_JAR, fs, libPath, "lib", false); Path dependencyLibTarGzip = fs.getDependencyTarGzip(); - if (fs.isFile(dependencyLibTarGzip)) { + if (actionDependency(null, false) == EXIT_SUCCESS) { LOG.info("Loading lib tar from " + dependencyLibTarGzip); fs.submitTarGzipAndUpdate(localResources); } else { @@ -1144,13 +1144,14 @@ public int actionDependency(String destinationFolder, boolean overwrite) String currentUser = RegistryUtils.currentUser(); LOG.info("Running command as user {}", currentUser); + Path dependencyLibTarGzip; if (destinationFolder == null) { - destinationFolder = String.format(YarnServiceConstants.DEPENDENCY_DIR, - VersionInfo.getVersion()); + dependencyLibTarGzip = fs.getDependencyTarGzip(); + } else { + dependencyLibTarGzip = new Path(destinationFolder, + YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_NAME + + YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_EXT); } - Path dependencyLibTarGzip = new Path(destinationFolder, - YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_NAME - + YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_EXT); // Check if dependency has already been uploaded, in which case log // appropriately and exit success (unless overwrite has been requested) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java index 5c2bac6897f..50b22e51107 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java @@ -28,6 +28,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.util.VersionInfo; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; @@ -363,6 +364,12 @@ public Path getDependencyTarGzip() { if (configuredDependencyTarballPath != null) { dependencyLibTarGzip = new Path(configuredDependencyTarballPath); } + if (dependencyLibTarGzip == null) { + dependencyLibTarGzip = new Path(String.format(YarnServiceConstants + .DEPENDENCY_DIR, VersionInfo.getVersion()), + YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_NAME + + YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_EXT); + } return dependencyLibTarGzip; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java index a95818f083d..db56ddc9382 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java @@ -20,6 +20,7 @@ import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.util.ToolRunner; import org.apache.hadoop.yarn.client.api.AppAdminClient; import org.apache.hadoop.yarn.client.cli.ApplicationCLI; @@ -27,6 +28,7 @@ import org.apache.hadoop.yarn.service.api.records.Component; import org.apache.hadoop.yarn.service.api.records.Service; import org.apache.hadoop.yarn.service.conf.ExampleAppJson; +import org.apache.hadoop.yarn.service.conf.YarnServiceConstants; import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; import org.apache.hadoop.yarn.service.utils.SliderFileSystem; import org.junit.After; @@ -41,6 +43,7 @@ import java.util.Arrays; import java.util.List; +import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.DEPENDENCY_TARBALL_PATH; import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_BASE_PATH; public class TestServiceCLI { @@ -51,6 +54,8 @@ private File basedir; private SliderFileSystem fs; private String basedirProp; + private File dependencyTarGz; + private String dependencyTarGzProp; private void runCLI(String[] args) throws Exception { LOG.info("running CLI: yarn {}", Arrays.asList(args)); @@ -85,12 +90,20 @@ public void setup() throws Throwable { basedir = new File("target", "apps"); basedirProp = YARN_SERVICE_BASE_PATH + "=" + basedir.getAbsolutePath(); conf.set(YARN_SERVICE_BASE_PATH, basedir.getAbsolutePath()); + dependencyTarGz = new File(basedir, YarnServiceConstants + .DEPENDENCY_TAR_GZ_FILE_NAME + YarnServiceConstants + .DEPENDENCY_TAR_GZ_FILE_EXT); + dependencyTarGzProp = DEPENDENCY_TARBALL_PATH + "=" + dependencyTarGz + .getAbsolutePath(); + conf.set(DEPENDENCY_TARBALL_PATH, dependencyTarGz.getAbsolutePath()); fs = new SliderFileSystem(conf); if (basedir.exists()) { FileUtils.deleteDirectory(basedir); } else { basedir.mkdirs(); } + System.setProperty(YarnServiceConstants.PROPERTY_LIB_DIR, basedir + .getAbsolutePath()); } @After @@ -114,6 +127,34 @@ public void testFlexComponents() throws Throwable { checkApp(serviceName, "master", 1L, 1000L, "qname"); } + @Test + public void testEnableFastLaunch() throws Exception { + fs.getFileSystem().create(new Path(basedir.getAbsolutePath(), "test.jar")) + .close(); + + Path defaultPath = new Path(dependencyTarGz.getAbsolutePath()); + Assert.assertFalse("Dependency tarball should not exist before the test", + fs.isFile(defaultPath)); + String[] args = {"app", "-D", dependencyTarGzProp, "-enableFastLaunch", + "-appTypes", AppAdminClient.UNIT_TEST_TYPE}; + runCLI(args); + Assert.assertTrue("Dependency tarball did not exist after the test", + fs.isFile(defaultPath)); + + File secondBaseDir = new File(basedir, "2"); + Path secondTarGz = new Path(new File(secondBaseDir, YarnServiceConstants + .DEPENDENCY_TAR_GZ_FILE_NAME + YarnServiceConstants + .DEPENDENCY_TAR_GZ_FILE_EXT).getAbsolutePath()); + Assert.assertFalse("Dependency tarball should not exist before the test", + fs.isFile(secondTarGz)); + String[] args2 = {"app", "-D", dependencyTarGzProp, "-enableFastLaunch", + secondBaseDir.getAbsolutePath(), "-appTypes", + AppAdminClient.UNIT_TEST_TYPE}; + runCLI(args2); + Assert.assertTrue("Dependency tarball did not exist after the test", + fs.isFile(secondTarGz)); + } + private void checkApp(String serviceName, String compName, long count, Long lifetime, String queue) throws IOException { Service service = ServiceApiUtil.loadService(fs, serviceName); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md index fb1375a6b47..3f24a96d5be 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md @@ -100,7 +100,7 @@ System-wide service AM properties can only be configured in the cluster `yarn-si | System-Level Config Name | Description | | ------------ | ------------- | -|yarn.service.framework.path | HDFS parent directory where the service AM dependency tarball can be found.| +|yarn.service.framework.path | HDFS URI of the service AM dependency tarball. When no file exists at this location, AM dependencies will be uploaded each time a service is started or launched. If unspecified, value will be assumed to be /yarn-services/${hadoop.version}/service-dep.tar.gz.| |yarn.service.base.path | HDFS parent directory where service artifacts will be stored (default ${user_home_dir}/.yarn/). |yarn.service.client-am.retry.max-wait-ms | Max retry time in milliseconds for the service client to talk to the service AM (default 900000, i.e. 15 minutes).| |yarn.service.client-am.retry-interval-ms | Retry interval in milliseconds for the service client to talk to the service AM (default 2000, i.e. 2 seconds).|