commit 39a44fbf64cd04af9de6ebfde0a23157205dc6d7 Author: Eric Yang Date: Mon Aug 20 19:57:16 2018 -0400 YARN-8569. First draft for yarn sysfs. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java index b63fe61..0db4d63 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java @@ -251,7 +251,14 @@ * Final, Docker run support ENTRY_POINT. */ YARN_CONTAINER_RUNTIME_DOCKER_RUN_OVERRIDE_DISABLE( - "YARN_CONTAINER_RUNTIME_DOCKER_RUN_OVERRIDE_DISABLE"); + "YARN_CONTAINER_RUNTIME_DOCKER_RUN_OVERRIDE_DISABLE"), + + /** + * $YARN_CONTAINER_RUNTIME_YARN_SYSFS + * Final, expose cluster information to container. + */ + YARN_CONTAINER_RUNTIME_YARN_SYSFS( + "YARN_CONTAINER_RUNTIME_YARN_SYSFS"); private final String variable; private Environment(String variable) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java index 5668d9f..00bbe7e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java @@ -70,6 +70,8 @@ import org.apache.hadoop.yarn.service.api.records.Container; import org.apache.hadoop.yarn.service.api.records.ContainerState; import org.apache.hadoop.yarn.service.api.records.Component; +import org.apache.hadoop.yarn.service.api.records.ConfigFile; +import org.apache.hadoop.yarn.service.api.records.ConfigFile.TypeEnum; import org.apache.hadoop.yarn.service.api.records.Service; import org.apache.hadoop.yarn.service.api.records.ServiceState; import org.apache.hadoop.yarn.service.conf.SliderExitCodes; @@ -95,6 +97,7 @@ import org.slf4j.LoggerFactory; import java.io.File; +import java.io.FileWriter; import java.io.IOException; import java.net.InetSocketAddress; import java.net.URI; @@ -851,6 +854,8 @@ ApplicationId submitApp(Service app) throws IOException, YarnException { addJarResource(serviceName, localResources); // add keytab if in secure env addKeytabResourceIfSecure(fs, localResources, app); + // add yarn sysfs to localResources + addYarnSysFs(appRootDir, localResources, app); if (LOG.isDebugEnabled()) { printLocalResources(localResources); } @@ -885,6 +890,40 @@ ApplicationId submitApp(Service app) throws IOException, YarnException { return submissionContext.getApplicationId(); } + private void addYarnSysFs(Path path, + Map localResources, Service app) + throws IOException { + List componentsWithYarnSysFS = new ArrayList(); + for(Component c : app.getComponents()) { + boolean enabled = Boolean.parseBoolean(c.getConfiguration() + .getEnv(ApplicationConstants.Environment + .YARN_CONTAINER_RUNTIME_YARN_SYSFS.name())); + if (enabled) { + componentsWithYarnSysFS.add(c); + } + } + if(componentsWithYarnSysFS.size() == 0) { + return; + } + String buffer = ServiceApiUtil.jsonSerDeser.toJson(app); + File localFile = new File("service.json"); + localFile.createNewFile(); + FileWriter writer = new FileWriter(localFile); + writer.write(buffer); + writer.close(); + LocalResource localResource = + fs.submitFile(localFile, path, ".", localFile.getName()); + Path serviceJson = new Path(path, "service.json"); + for (Component c : componentsWithYarnSysFS) { + ConfigFile e = new ConfigFile(); + e.type(TypeEnum.TEMPLATE); + e.srcFile(serviceJson.toString()); + e.destFile("/hadoop/yarn/sysfs/service.json"); + c.getConfiguration().getFiles().add(e); + } + localResources.put("service.json", localResource); + } + private void setLogAggregationContext(Service app, Configuration conf, ApplicationSubmissionContext submissionContext) { LogAggregationContext context = Records.newRecord(LogAggregationContext