commit 7be9c31084b3dc613ee1db85a5ab9bd401a5f907 Author: Wangda Tan Date: Tue Apr 24 15:18:12 2018 -0700 YARN-8079, localize file support in NS (006 Change-Id: I6f8c992bb7714f9c2018f5cc2810c6513c8ec066 diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml index 849f886cd73..f4855729653 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml @@ -483,6 +483,8 @@ definitions: - YAML - TEMPLATE - HADOOP_XML + - STATIC + - ARCHIVE dest_file: type: string description: The path that this configuration file should be created as. If it is an absolute path, it will be mounted into the DOCKER container. Absolute paths are only allowed for DOCKER containers. If it is a relative path, only the file name should be provided, and the file will be created in the container local working directory under a folder named conf. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Component.java index 7179b86975d..78075686a47 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Component.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Component.java @@ -102,7 +102,6 @@ private List containers = Collections.synchronizedList(new ArrayList()); - @JsonProperty("restart_policy") @XmlElement(name = "restart_policy") private RestartPolicyEnum restartPolicy = RestartPolicyEnum.ALWAYS; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java index d3b18bc3d2b..623feedb11f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java @@ -55,7 +55,8 @@ @XmlEnum public enum TypeEnum { XML("XML"), PROPERTIES("PROPERTIES"), JSON("JSON"), YAML("YAML"), TEMPLATE( - "TEMPLATE"), HADOOP_XML("HADOOP_XML"); + "TEMPLATE"), HADOOP_XML("HADOOP_XML"), STATIC("STATIC"), ARCHIVE( + "ARCHIVE"); private String value; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConstants.java index 7b474f6e249..c6d7f1a448f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConstants.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConstants.java @@ -84,6 +84,7 @@ String HADOOP_USER_NAME = "HADOOP_USER_NAME"; String APP_CONF_DIR = "conf"; + String APP_LOCALIZED_DIR = "localized"; String APP_LIB_DIR = "lib"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java index 26c332b44bc..9cbacee2173 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java @@ -86,8 +86,9 @@ public void validateConfigFiles(List configFiles, if (file.getType() == null) { throw new IllegalArgumentException("File type is empty"); } + ConfigFile.TypeEnum fileType = file.getType(); - if (file.getType().equals(ConfigFile.TypeEnum.TEMPLATE)) { + if (fileType.equals(ConfigFile.TypeEnum.TEMPLATE)) { if (StringUtils.isEmpty(file.getSrcFile()) && !file.getProperties().containsKey(CONTENT)) { throw new IllegalArgumentException(MessageFormat.format("For {0} " + @@ -96,6 +97,18 @@ public void validateConfigFiles(List configFiles, "the 'properties' field of ConfigFile. ", ConfigFile.TypeEnum.TEMPLATE, CONTENT)); } + } else if (fileType.equals(ConfigFile.TypeEnum.STATIC) || fileType.equals( + ConfigFile.TypeEnum.ARCHIVE)) { + if (!file.getProperties().isEmpty()) { + throw new IllegalArgumentException(String + .format("For %s format, should not specify any 'properties.'", + fileType)); + } + if (file.getSrcFile() == null || file.getSrcFile().isEmpty()) { + throw new IllegalArgumentException(String.format( + "For %s format, should make sure that srcFile is specified", + fileType)); + } } if (!StringUtils.isEmpty(file.getSrcFile())) { Path p = new Path(file.getSrcFile()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java index ee276866afb..5beb9529419 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java @@ -96,6 +96,10 @@ public void buildContainerLaunchContext(AbstractLauncher launcher, ProviderUtils.createConfigFileAndAddLocalResource(launcher, fileSystem, component, tokensForSubstitution, instance, context); + // handles static files (like normal file / archive file) for localization. + ProviderUtils.handleStaticFilesForLocalization(launcher, fileSystem, + component); + // substitute launch command String launchCommand = component.getLaunchCommand(); // docker container may have empty commands diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java index d65a1969a13..4f84769b61e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.service.provider; import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsAction; @@ -170,6 +171,74 @@ public static Path initCompInstanceDir(SliderFileSystem fs, // 1. Create all config files for a component on hdfs for localization // 2. Add the config file to localResource + public static synchronized void handleStaticFilesForLocalization( + AbstractLauncher launcher, SliderFileSystem fs, Component component) + throws IOException { + for (ConfigFile staticFile : component.getConfiguration().getFiles()) { + // Only handle static file here. + if (!isStaticFile(staticFile)) { + continue; + } + + if (staticFile.getSrcFile() == null) { + // This should not happen, AbstractClientProvider should have checked + // this. + throw new IOException("srcFile is null, please double check."); + } + Path sourceFile = new Path(staticFile.getSrcFile()); + + // Output properties to sourceFile if not existed + if (!fs.getFileSystem().exists(sourceFile)) { + throw new IOException( + "srcFile=" + sourceFile + " doesn't exist, please double check."); + } + + FileStatus fileStatus = fs.getFileSystem().getFileStatus(sourceFile); + if (fileStatus.isDirectory()) { + throw new IOException("srcFile=" + sourceFile + + " is a directory, which is not supported."); + } + + // Add resource for localization + LocalResource localResource = fs.createAmResource(sourceFile, + (staticFile.getType() == ConfigFile.TypeEnum.ARCHIVE ? + LocalResourceType.ARCHIVE : + LocalResourceType.FILE)); + Path destFile = new Path(sourceFile.getName()); + if (staticFile.getDestFile() != null && !staticFile.getDestFile() + .isEmpty()) { + destFile = new Path(staticFile.getDestFile()); + } + + String symlink = APP_LOCALIZED_DIR + "/" + destFile.getName(); + addLocalResource(launcher, symlink, localResource, destFile); + } + } + + private static void addLocalResource(AbstractLauncher launcher, + String symlink, LocalResource localResource, Path destFile) { + if (destFile.isAbsolute()) { + launcher.addLocalResource(symlink, localResource, destFile.toString()); + log.info("Add config file for localization: " + symlink + " -> " + + localResource.getResource().getFile() + ", dest mount path: " + + destFile); + } else{ + launcher.addLocalResource(symlink, localResource); + log.info("Add config file for localization: " + symlink + " -> " + + localResource.getResource().getFile()); + } + } + + // Static file is files uploaded by users before launch the service. Which + // should be localized to container local disk without any changes. + private static boolean isStaticFile(ConfigFile file) { + return file.getType().equals(ConfigFile.TypeEnum.ARCHIVE) || file.getType() + .equals(ConfigFile.TypeEnum.STATIC); + } + + + // 1. Create all config files for a component on hdfs for localization + // 2. Add the config file to localResource public static synchronized void createConfigFileAndAddLocalResource( AbstractLauncher launcher, SliderFileSystem fs, Component component, Map tokensForSubstitution, ComponentInstance instance, @@ -190,6 +259,10 @@ public static synchronized void createConfigFileAndAddLocalResource( } for (ConfigFile originalFile : component.getConfiguration().getFiles()) { + if (isStaticFile(originalFile)) { + continue; + } + ConfigFile configFile = originalFile.copy(); String fileName = new Path(configFile.getDestFile()).getName(); @@ -199,7 +272,11 @@ public static synchronized void createConfigFileAndAddLocalResource( .replaceAll(Pattern.quote(token.getKey()), token.getValue())); } + // When source file not specified, upload new configs to + // {{compInstanceDir/fileName}} + // Otherwise, use sourceFile. Path remoteFile = new Path(compInstanceDir, fileName); + // Output properties to sourceFile if not existed. if (!fs.getFileSystem().exists(remoteFile)) { log.info("Saving config file on hdfs for component " + instance .getCompInstanceName() + ": " + configFile); @@ -231,19 +308,9 @@ public static synchronized void createConfigFileAndAddLocalResource( // Add resource for localization LocalResource configResource = fs.createAmResource(remoteFile, LocalResourceType.FILE); - File destFile = new File(configFile.getDestFile()); + Path destFile = new Path(configFile.getDestFile()); String symlink = APP_CONF_DIR + "/" + fileName; - if (destFile.isAbsolute()) { - launcher.addLocalResource(symlink, configResource, - configFile.getDestFile()); - log.info("Add config file for localization: " + symlink + " -> " - + configResource.getResource().getFile() + ", dest mount path: " - + configFile.getDestFile()); - } else { - launcher.addLocalResource(symlink, configResource); - log.info("Add config file for localization: " + symlink + " -> " - + configResource.getResource().getFile()); - } + addLocalResource(launcher, symlink, configResource, destFile); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestProviderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestProviderUtils.java new file mode 100644 index 00000000000..55b397525bb --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestProviderUtils.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service.provider; + +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.yarn.api.records.LocalResource; +import org.apache.hadoop.yarn.api.records.LocalResourceType; +import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; +import org.apache.hadoop.yarn.api.records.URL; +import org.apache.hadoop.yarn.service.api.records.Component; +import org.apache.hadoop.yarn.service.api.records.ConfigFile; +import org.apache.hadoop.yarn.service.api.records.Configuration; +import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher; +import org.apache.hadoop.yarn.service.utils.SliderFileSystem; +import org.junit.Test; +import org.mockito.Mockito; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Test functionality of ProviderUtils. + */ +public class TestProviderUtils { + @Test + public void testStaticFileLocalization() throws IOException { + // A bunch of mocks ... + Component comp = mock(Component.class); + AbstractLauncher launcher = mock(AbstractLauncher.class); + SliderFileSystem sfs = mock(SliderFileSystem.class); + FileSystem fs = mock(FileSystem.class); + when(fs.getFileStatus(any(Path.class))).thenAnswer( + invocationOnMock -> new FileStatus(1L, false, 1, 1L, 1L, + (Path) invocationOnMock.getArguments()[0])); + when(fs.exists(any(Path.class))).thenReturn(true); + when(sfs.getFileSystem()).thenReturn(fs); + Configuration conf = mock(Configuration.class); + List configFileList = new ArrayList<>(); + when(conf.getFiles()).thenReturn(configFileList); + when(comp.getConfiguration()).thenReturn(conf); + when(sfs.createAmResource(any(Path.class), any(LocalResourceType.class))) + .thenAnswer(invocationOnMock -> new LocalResource() { + @Override + public URL getResource() { + return URL.fromPath(((Path) invocationOnMock.getArguments()[0])); + } + + @Override + public void setResource(URL resource) { + + } + + @Override + public long getSize() { + return 0; + } + + @Override + public void setSize(long size) { + + } + + @Override + public long getTimestamp() { + return 0; + } + + @Override + public void setTimestamp(long timestamp) { + + } + + @Override + public LocalResourceType getType() { + return (LocalResourceType) invocationOnMock.getArguments()[1]; + } + + @Override + public void setType(LocalResourceType type) { + + } + + @Override + public LocalResourceVisibility getVisibility() { + return null; + } + + @Override + public void setVisibility(LocalResourceVisibility visibility) { + + } + + @Override + public String getPattern() { + return null; + } + + @Override + public void setPattern(String pattern) { + + } + + @Override + public boolean getShouldBeUploadedToSharedCache() { + return false; + } + + @Override + public void setShouldBeUploadedToSharedCache( + boolean shouldBeUploadedToSharedCache) { + + } + }); + + // Initialize list of files. + configFileList.add(new ConfigFile().srcFile("hdfs://default/sourceFile1") + .destFile("destFile1").type(ConfigFile.TypeEnum.ARCHIVE)); + configFileList.add(new ConfigFile().srcFile("hdfs://default/sourceFile2") + .destFile("folder/destFile_2").type(ConfigFile.TypeEnum.STATIC)); + configFileList.add(new ConfigFile().srcFile("hdfs://default/sourceFile3") + .destFile("destFile3").type(ConfigFile.TypeEnum.JSON)); + configFileList.add(new ConfigFile().srcFile("hdfs://default/sourceFile4") + .type(ConfigFile.TypeEnum.STATIC)); + + ProviderUtils.handleStaticFilesForLocalization(launcher, sfs, comp); + Mockito.verify(launcher).addLocalResource(Mockito.eq("localized/destFile1"), + any(LocalResource.class)); + Mockito.verify(launcher).addLocalResource( + Mockito.eq("localized/destFile_2"), any(LocalResource.class)); + Mockito.verify(launcher).addLocalResource( + Mockito.eq("localized/sourceFile4"), any(LocalResource.class)); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md index 6bb407a1e9d..2b950eecdec 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md @@ -251,7 +251,11 @@ A config file that needs to be created and made available as a volume in a servi |Name|Description|Required|Schema|Default| |----|----|----|----|----| +<<<<<<< ours |type|Config file in the standard format like xml, properties, json, yaml, template.|false|enum (XML, PROPERTIES, JSON, YAML, TEMPLATE, HADOOP_XML)|| +======= +|type|Config file in the standard format like xml, properties, json, yaml, template. When STATIC/ARCHIVE is specified, file must be uploaded to remote file system before launch the job, and YARN service framework will localize files prior to launch containers|false|enum (XML, PROPERTIES, JSON, YAML, TEMPLATE, ENV, HADOOP_XML,STATIC,ARCHIVE)|| +>>>>>>> theirs |dest_file|The path that this configuration file should be created as. If it is an absolute path, it will be mounted into the DOCKER container. Absolute paths are only allowed for DOCKER containers. If it is a relative path, only the file name should be provided, and the file will be created in the container local working directory under a folder named conf.|false|string|| |src_file|This provides the source location of the configuration file, the content of which is dumped to dest_file post property substitutions, in the format as specified in type. Typically the src_file would point to a source controlled network accessible file maintained by tools like puppet, chef, or hdfs etc. Currently, only hdfs is supported.|false|string|| |properties|A blob of key value pairs that will be dumped in the dest_file in the format as specified in type. If src_file is specified, src_file content are dumped in the dest_file and these properties will overwrite, if any, existing properties in src_file or be added as new properties in src_file.|false|object||