commit 90fa9951218a4705a3ca6dc2ad036f7564361ad8 Author: Eric Yang Date: Mon Aug 20 19:57:16 2018 -0400 YARN-8569. Added YARN sysfs interface for provide cluster information. Contributed by Eric Yang diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java index b63fe61..0db4d63 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java @@ -251,7 +251,14 @@ * Final, Docker run support ENTRY_POINT. */ YARN_CONTAINER_RUNTIME_DOCKER_RUN_OVERRIDE_DISABLE( - "YARN_CONTAINER_RUNTIME_DOCKER_RUN_OVERRIDE_DISABLE"); + "YARN_CONTAINER_RUNTIME_DOCKER_RUN_OVERRIDE_DISABLE"), + + /** + * $YARN_CONTAINER_RUNTIME_YARN_SYSFS + * Final, expose cluster information to container. + */ + YARN_CONTAINER_RUNTIME_YARN_SYSFS( + "YARN_CONTAINER_RUNTIME_YARN_SYSFS"); private final String variable; private Environment(String variable) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java index 0caa119..6cf5190 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java @@ -45,6 +45,9 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager; +import org.apache.hadoop.yarn.service.api.records.Component; +import org.apache.hadoop.yarn.service.api.records.Container; +import org.apache.hadoop.yarn.service.api.records.Service; import org.apache.hadoop.yarn.service.api.records.ServiceState; import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException; import org.apache.hadoop.yarn.service.monitor.ServiceMonitor; @@ -60,7 +63,9 @@ import java.net.URISyntaxException; import java.nio.ByteBuffer; import java.security.PrivilegedExceptionAction; +import java.util.ArrayList; import java.util.Iterator; +import java.util.List; import java.util.Map; import static org.apache.hadoop.yarn.service.conf.YarnServiceConstants.KEYTAB_LOCATION; @@ -302,6 +307,12 @@ public static synchronized void checkAndUpdateServiceState( LOG.info("Service state changed from {} -> {}", curState, scheduler.getApp().getState()); } + populateYarnSysFS(scheduler); + } + + private static void populateYarnSysFS(ServiceScheduler scheduler) { + Service service = scheduler.getApp(); + scheduler.syncSysFs(service); } private void printSystemEnv() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java index 0801ad0..aeddab7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java @@ -21,6 +21,9 @@ import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.WebResource.Builder; + import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; @@ -76,6 +79,7 @@ import org.apache.hadoop.yarn.service.registry.YarnRegistryViewForProviders; import org.apache.hadoop.yarn.service.timelineservice.ServiceMetricsSink; import org.apache.hadoop.yarn.service.timelineservice.ServiceTimelinePublisher; +import org.apache.hadoop.yarn.service.utils.HttpUtil; import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; import org.apache.hadoop.yarn.service.utils.ServiceRegistryUtils; import org.apache.hadoop.yarn.service.utils.ServiceUtils; @@ -948,4 +952,47 @@ public synchronized void terminateServiceIfAllComponentsFinished() { public ServiceUtils.ProcessTerminationHandler getTerminationHandler() { return terminationHandler; } + + public void syncSysFs(Service app) { + boolean success = true; + Configuration conf = getConfig(); + String spec; + try { + String port = conf.get("yarn.nodemanager.webapp.address").split(":")[1]; + spec = ServiceApiUtil.jsonSerDeser.toJson(app); + for (org.apache.hadoop.yarn.service.api.records.Component c : + app.getComponents()) { + for (org.apache.hadoop.yarn.service.api.records.Container container : + c.getContainers()) { + String bareHost = container.getBareHost(); + StringBuilder requestPath = new StringBuilder(); + if (YarnConfiguration.useHttps(conf)) { + requestPath.append("https://"); + } else { + requestPath.append("http://"); + } + requestPath.append(bareHost); + requestPath.append(":"); + requestPath.append(port); + requestPath.append("/ws/v1/node/yarn/sysfs/"); + requestPath.append(UserGroupInformation.getCurrentUser().getShortUserName()); + requestPath.append("/"); + requestPath.append(app.getId()); + requestPath.append("/"); + requestPath.append(container.getId()); + Builder builder = HttpUtil.connect(requestPath.toString()); + ClientResponse response = builder.put(ClientResponse.class, spec); + if (response.getStatus()!=ClientResponse.Status.OK.getStatusCode()) { + LOG.warn("Error synchronize YARN sysfs: " + response.getEntity(String.class)); + success = false; + } + } + } + if (success) { + LOG.info("YARN sysfs synchronized."); + } + } catch (Exception e) { + LOG.error("Fail to sync service spec: {}", e); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/HttpUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/HttpUtil.java new file mode 100644 index 0000000..4e0ddaf --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/HttpUtil.java @@ -0,0 +1,96 @@ +package org.apache.hadoop.yarn.service.utils; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.security.PrivilegedExceptionAction; + +import javax.ws.rs.core.HttpHeaders; +import javax.ws.rs.core.MediaType; + +import org.apache.commons.codec.binary.Base64; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.client.AuthenticationException; +import org.apache.hadoop.security.authentication.util.KerberosUtil; +import org.ietf.jgss.GSSContext; +import org.ietf.jgss.GSSException; +import org.ietf.jgss.GSSManager; +import org.ietf.jgss.GSSName; +import org.ietf.jgss.Oid; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.sun.jersey.api.client.Client; +import com.sun.jersey.api.client.WebResource.Builder; + +public class HttpUtil { + private static final Logger LOG = + LoggerFactory.getLogger(HttpUtil.class); + private static final Base64 BASE_64_CODEC = new Base64(0); + + + /** + * Generate SPNEGO challenge request token. + * + * @param server - hostname to contact + * @throws IOException + * @throws InterruptedException + */ + public static String generateToken(String server) throws IOException, InterruptedException { + UserGroupInformation currentUser = UserGroupInformation.getCurrentUser(); + LOG.debug("The user credential is {}", currentUser); + String challenge = currentUser + .doAs(new PrivilegedExceptionAction() { + @Override + public String run() throws Exception { + try { + // This Oid for Kerberos GSS-API mechanism. + Oid mechOid = KerberosUtil.getOidInstance("GSS_KRB5_MECH_OID"); + GSSManager manager = GSSManager.getInstance(); + // GSS name for server + GSSName serverName = manager.createName("HTTP@" + server, + GSSName.NT_HOSTBASED_SERVICE); + // Create a GSSContext for authentication with the service. + // We're passing client credentials as null since we want them to + // be read from the Subject. + GSSContext gssContext = manager.createContext( + serverName.canonicalize(mechOid), mechOid, null, + GSSContext.DEFAULT_LIFETIME); + gssContext.requestMutualAuth(true); + gssContext.requestCredDeleg(true); + // Establish context + byte[] inToken = new byte[0]; + byte[] outToken = gssContext.initSecContext(inToken, 0, + inToken.length); + gssContext.dispose(); + // Base64 encoded and stringified token for server + LOG.debug("Got valid challenge for host {}", serverName); + return new String(BASE_64_CODEC.encode(outToken), + StandardCharsets.US_ASCII); + } catch (GSSException | IllegalAccessException + | NoSuchFieldException | ClassNotFoundException e) { + LOG.error("Error: {}", e); + throw new AuthenticationException(e); + } + } + }); + return challenge; + } + + public static Builder connect(String url) throws URISyntaxException, + IOException, InterruptedException { + boolean useKerberos = UserGroupInformation.isSecurityEnabled(); + URI resource = new URI(url); + Client client = Client.create(); + Builder builder = client + .resource(url).type(MediaType.APPLICATION_JSON); + if (useKerberos) { + String challenge = generateToken(resource.getHost()); + builder.header(HttpHeaders.AUTHORIZATION, "Negotiate " + + challenge); + LOG.debug("Authorization: Negotiate {}", challenge); + } + return builder; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java index 9219569..52530fb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java @@ -24,17 +24,22 @@ import com.google.common.collect.Multimap; import com.google.common.collect.Sets; import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.registry.client.api.RegistryConstants; import org.apache.hadoop.registry.client.binding.RegistryUtils; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.api.ApplicationConstants; +import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.service.api.records.ComponentState; +import org.apache.hadoop.yarn.service.api.records.ConfigFile; import org.apache.hadoop.yarn.service.api.records.Container; import org.apache.hadoop.yarn.service.api.records.ContainerState; import org.apache.hadoop.yarn.service.api.records.Service; +import org.apache.hadoop.yarn.service.api.records.ConfigFile.TypeEnum; import org.apache.hadoop.yarn.service.api.records.Artifact; import org.apache.hadoop.yarn.service.api.records.Component; import org.apache.hadoop.yarn.service.api.records.Configuration; @@ -52,11 +57,14 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; +import java.io.FileWriter; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; @@ -65,6 +73,7 @@ import static org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages.ERROR_COMP_DOES_NOT_NEED_UPGRADE; import static org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages.ERROR_COMP_INSTANCE_DOES_NOT_NEED_UPGRADE; +import static org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher.ENV_DOCKER_CONTAINER_MOUNTS; public class ServiceApiUtil { private static final Logger LOG = @@ -87,7 +96,7 @@ private static final PatternValidator userNamePattern = new PatternValidator("[a-z][a-z0-9-.]*"); - + private static final String YARN_SYSFS_MOUNT = "sysfs:/hadoop/yarn/sysfs:ro"; @VisibleForTesting public static void setJsonSerDeser(JsonSerDeser jsd) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceAM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceAM.java index 21e93fa..14daff5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceAM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceAM.java @@ -18,7 +18,6 @@ package org.apache.hadoop.yarn.service; -import com.google.common.base.Supplier; import com.google.common.collect.ImmutableMap; import org.apache.commons.io.FileUtils; import org.apache.curator.test.TestingCluster; @@ -426,4 +425,34 @@ public void testContainersReleasedWhenPreLaunchFails() am.getComponent("compa").getPendingInstances().size()); am.stop(); } + + @Test(timeout = 30000) + public void testSyncSysFS() { + ApplicationId applicationId = ApplicationId.newInstance( + System.currentTimeMillis(), 1); + Service exampleApp = new Service(); + exampleApp.setId(applicationId.toString()); + exampleApp.setVersion("v1"); + exampleApp.setName("tensorflow"); + + Component compA = createComponent("compa", 1, "pwd"); + compA.getConfiguration().getEnv().put("YARN_CONTAINER_RUNTIME_YARN_SYSFS", "true"); + Artifact artifact = new Artifact(); + artifact.setType(Artifact.TypeEnum.TARBALL); + compA.artifact(artifact); + exampleApp.addComponent(compA); + try { + MockServiceAM am = new MockServiceAM(exampleApp); + am.init(conf); + am.start(); + ServiceScheduler scheduler = am.context.scheduler; + scheduler.syncSysFs(exampleApp); + scheduler.close(); + am.stop(); + am.close(); + } catch (Exception e) { + LOG.error("Fail to sync sysfs: {}", e); + Assert.fail("Fail to sync sysfs."); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java index 9b604ce..58ee121 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java @@ -51,6 +51,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerDiagnosticsUpdateEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationException; import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException; import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerPrepareContext; import org.apache.hadoop.yarn.server.nodemanager.util.NodeManagerHardwareUtils; @@ -244,6 +245,21 @@ public abstract boolean isContainerAlive(ContainerLivenessContext ctx) throws IOException; /** + * Update cluster information inside container. + * + * @param ctx ContainerRuntimeContext + * @param user Owner of application + * @param appId YARN application ID + * @param containerId Container ID + * @param spec cluster information + * @throws IOException if there is a failure while writing spec to disk + * @throws PrivilegedOperationException if no access to perform operation + */ + public abstract void updateYarnSysFS(Context ctx, String user, + String appId, String containerId, String spec) throws IOException, + PrivilegedOperationException; + + /** * Recover an already existing container. This is a blocking call and returns * only when the container exits. Note that the container must have been * activated prior to this call. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java index 27224a5..3b3ce86 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java @@ -42,6 +42,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.UnsupportedFileSystemException; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.service.ServiceStateException; import org.apache.hadoop.util.Shell; import org.apache.hadoop.util.Shell.CommandExecutor; import org.apache.hadoop.util.Shell.ExitCodeException; @@ -54,7 +55,9 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerDiagnosticsUpdateEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationException; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeContext; import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerLivenessContext; import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerReapContext; import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerSignalContext; @@ -1008,4 +1011,11 @@ public void clearLogDirPermissions() { } return paths; } + + @Override + public void updateYarnSysFS(Context ctx, String user, + String appId, String containerId, String spec) throws IOException, + PrivilegedOperationException { + throw new ServiceStateException("Implementation unavailable"); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java index f75ead2..04d838a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java @@ -61,6 +61,7 @@ import org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler; import org.apache.hadoop.yarn.server.nodemanager.util.LCEResourcesHandler; import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; import java.net.InetSocketAddress; import java.util.ArrayList; @@ -930,6 +931,23 @@ public void mountCgroups(List cgroupKVs, String hierarchy) } @VisibleForTesting + void updateYarnSysFs(String appId, String localDir) + throws IOException { + try { + PrivilegedOperationExecutor privOpExecutor = + PrivilegedOperationExecutor.getInstance(super.getConf()); + privOpExecutor.executeUpdateYarnSysFs(localDir); + } catch (PrivilegedOperationException e) { + int exitCode = e.getExitCode(); + LOG.warn("Exception in LinuxContainerExecutor updateYarnSysFs ", e); + + throw new IOException("Problem updating Yarn SysFs " + appId + + "; exit code = " + exitCode + " and output: " + e.getOutput(), + e); + } + } + + @VisibleForTesting public ResourceHandler getResourceHandler() { return resourceHandlerChain; } @@ -968,4 +986,36 @@ void postComplete(final ContainerId containerId) { "containerId: {}. Exception: ", containerId, e); } } + + @Override + public void updateYarnSysFS(Context ctx, String user, + String appId, String containerId, String spec) throws IOException, + PrivilegedOperationException { + LocalDirsHandlerService dirsHandler = nmContext.getLocalDirsHandler(); + Path sysFSPath = dirsHandler.getLocalPathForWrite( + "nmPrivate/" + appId + "/" + containerId + "/service.json"); + File file = new File(sysFSPath.toString()); + List localDirs = dirsHandler.getLocalDirs(); + if(file.createNewFile()) { + FileOutputStream output = new FileOutputStream(file); + output.write(spec.getBytes()); + output.close(); + } + PrivilegedOperation privOp = new PrivilegedOperation( + PrivilegedOperation.OperationType.SYNC_YARN_SYSFS); + + privOp.appendArgs(user, + UserGroupInformation.getLoginUser().getShortUserName(), + Integer.toString(PrivilegedOperation.RunAsUserCommand + .SYNC_YARN_SYSFS.getValue()), + appId, containerId, + StringUtils.join(PrivilegedOperation + .LINUX_FILE_PATH_SEPARATOR, localDirs)); + privOp.disableFailureLogging(); + PrivilegedOperationExecutor privilegedOperationExecutor = + PrivilegedOperationExecutor.getInstance(nmContext.getConf()); + privilegedOperationExecutor.executePrivilegedOperation(null, + privOp, null, null, false, false); + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperation.java index 92a82e8..f199662 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperation.java @@ -56,7 +56,8 @@ LIST_AS_USER(""), // no CLI switch supported yet. ADD_NUMA_PARAMS(""), // no CLI switch supported yet. REMOVE_DOCKER_CONTAINER("--remove-docker-container"), - INSPECT_DOCKER_CONTAINER("--inspect-docker-container"); + INSPECT_DOCKER_CONTAINER("--inspect-docker-container"), + SYNC_YARN_SYSFS(""); private final String option; @@ -153,7 +154,8 @@ public int hashCode() { SIGNAL_CONTAINER(2), DELETE_AS_USER(3), LAUNCH_DOCKER_CONTAINER(4), - LIST_AS_USER(5); + LIST_AS_USER(5), + SYNC_YARN_SYSFS(6); private int value; RunAsUserCommand(int value) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperationExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperationExecutor.java index 76949ff..89816b0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperationExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperationExecutor.java @@ -279,4 +279,22 @@ public String executePrivilegedOperation(PrivilegedOperation operation, return finalOp; } + + /** + * Update YARN sysfs, download and update service json in + * appcache. + * + * @param localDir + * @throws PrivilegedOperationException + */ + public void executeUpdateYarnSysFs(String localDir) + throws PrivilegedOperationException { + try { + File service = new File(localDir + "/service.json"); + service.createNewFile(); + } catch (Exception e) { + throw new PrivilegedOperationException("Invalid access to update yarn sysfs: " + + localDir); + } + } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java index 1872830..09711e4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java @@ -25,6 +25,7 @@ import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.server.nodemanager.Context; +import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerCommand; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerCommandExecutor; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerKillCommand; @@ -65,6 +66,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeContext; import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; import java.net.InetAddress; import java.net.UnknownHostException; @@ -175,6 +177,12 @@ * This feature is disabled by default. When this feature is disabled or set * to false, the container will be removed as soon as it exits. * + *
  • + * {@code YARN_CONTAINER_RUNTIME_YARN_SYSFS} allows export yarn service + * json to docker container. This feature is disabled by default. + * when this feature is set, service.json will be available in + * /hadoop/yarn/sysfs/service.json. + *
  • * */ @InterfaceAudience.Private @@ -222,6 +230,11 @@ @InterfaceAudience.Private public static final String ENV_DOCKER_CONTAINER_DELAYED_REMOVAL = "YARN_CONTAINER_RUNTIME_DOCKER_DELAYED_REMOVAL"; + @InterfaceAudience.Private + public static final String ENV_DOCKER_CONTAINER_YARN_SYSFS = + "YARN_CONTAINER_RUNTIME_YARN_SYSFS"; + public static final String YARN_SYSFS_PATH = + "/hadoop/yarn/sysfs"; private Configuration conf; private Context nmContext; private DockerClient dockerClient; @@ -836,6 +849,12 @@ public void launchContainer(ContainerRuntimeContext ctx) runCommand.addAllReadOnlyMountLocations(filecacheDirs); runCommand.addAllReadOnlyMountLocations(userFilecacheDirs); + if (environment.containsKey(ENV_DOCKER_CONTAINER_YARN_SYSFS)) { + String src = new Path(containerWorkDir, "sysfs").toString(); + String dst = YARN_SYSFS_PATH; + runCommand.addReadOnlyMountLocation(src, dst); + } + if (environment.containsKey(ENV_DOCKER_CONTAINER_MOUNTS)) { Matcher parsedMounts = USER_MOUNT_PATTERN.matcher( environment.get(ENV_DOCKER_CONTAINER_MOUNTS)); @@ -912,6 +931,11 @@ public void launchContainer(ContainerRuntimeContext ctx) addCGroupParentIfRequired(resourcesOpts, containerIdStr, runCommand); + if(environment.containsKey(ENV_DOCKER_CONTAINER_YARN_SYSFS) && + Boolean.parseBoolean(environment.get(ENV_DOCKER_CONTAINER_YARN_SYSFS))) { + runCommand.setYarnSysFS(true); + } + if (useEntryPoint) { runCommand.setOverrideDisabled(true); runCommand.addEnv(environment); @@ -1331,4 +1355,38 @@ private void addDockerClientConfigToRunCommand(ContainerRuntimeContext ctx, } } } + + public void handleYarnSysFSUpdate(ContainerRuntimeContext ctx, String user, + String appId, String containerId, String buffer) { + try { + LocalDirsHandlerService dirsHandler = nmContext.getLocalDirsHandler(); + org.apache.hadoop.fs.Path sysFSPath = dirsHandler.getLocalPathForWrite( + "nmPrivate/" + appId + "/" + containerId + "/service.json"); + File file = new File(sysFSPath.toString()); + List localDirs = dirsHandler.getLocalDirs(); + if(file.createNewFile()) { + FileOutputStream output = new FileOutputStream(file); + output.write(buffer.getBytes()); + output.close(); + } + PrivilegedOperation privOp = new PrivilegedOperation( + PrivilegedOperation.OperationType.SYNC_YARN_SYSFS); + + privOp.appendArgs(user, + UserGroupInformation.getLoginUser().getShortUserName(), + Integer.toString(PrivilegedOperation.RunAsUserCommand + .SYNC_YARN_SYSFS.getValue()), + appId, containerId, + org.apache.hadoop.util.StringUtils.join(PrivilegedOperation + .LINUX_FILE_PATH_SEPARATOR, localDirs)); + privOp.disableFailureLogging(); + PrivilegedOperationExecutor privilegedOperationExecutor = + PrivilegedOperationExecutor.getInstance(nmContext.getConf()); + privilegedOperationExecutor.executePrivilegedOperation(null, + privOp, null, null, false, false); + } catch (IOException | PrivilegedOperationException e) { + LOG.error("Fail to sync yarn sysfs for container: {}, reason: ", + containerId, e); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java index 48ada5a..e8b5d46 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java @@ -212,4 +212,10 @@ public boolean containsEnv() { public final void addEnv(Map environment) { userEnv.putAll(environment); } + + public DockerRunCommand setYarnSysFS(boolean toggle) { + String value = Boolean.toString(toggle); + super.addCommandArguments("use-yarn-sysfs", value); + return this; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java index bb0881b..7e7b1b5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java @@ -38,6 +38,7 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import javax.ws.rs.GET; +import javax.ws.rs.PUT; import javax.ws.rs.Path; import javax.ws.rs.PathParam; import javax.ws.rs.Produces; @@ -56,6 +57,7 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.http.JettyUtils; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.service.ServiceStateException; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -74,6 +76,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationException; import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.AppInfo; import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.AppsInfo; import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.ContainerInfo; @@ -554,6 +557,35 @@ public Object getNMResourceInfo( return new NMResourceInfo(); } + @PUT + @Path("/yarn/sysfs/{user}/{appId}/{containerId}") + @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, + MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 }) + public Response syncYarnSysFS(@javax.ws.rs.core.Context HttpServletRequest request, + @PathParam("user") String user, + @PathParam("appId") String appId, + @PathParam("containerId") String containerId, + String spec) { + if (UserGroupInformation.isSecurityEnabled()) { + if (!request.getRemoteUser().equals(user)) { + return Response.status(Status.FORBIDDEN).build(); + } + } + try { + nmContext.getContainerExecutor().updateYarnSysFS(nmContext, user, appId, + containerId, spec); + } catch (IOException | ServiceStateException e) { + LOG.error("Fail to sync yarn sysfs for container: {}, reason: ", + containerId, e); + return Response.status(Status.INTERNAL_SERVER_ERROR).build(); + } catch (PrivilegedOperationException e) { + LOG.error("Fail to sync yarn sysfs for container: {}, reason: ", + containerId, e); + return Response.status(Status.FORBIDDEN).build(); + } + return Response.ok().build(); + } + private long parseLongParam(String bytes) { if (bytes == null || bytes.isEmpty()) { return Long.MAX_VALUE; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c index f8b89ee..d975253 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c @@ -1543,6 +1543,27 @@ int create_user_filecache_dirs(const char * user, char* const* local_dirs) { return rc; } +int create_yarn_sysfs(const char* user, const char *app_id, + const char *container_id, const char *work_dir, char* const* local_dirs) { + int result = OUT_OF_MEMORY; + const mode_t perms = S_IRWXU | S_IXGRP; + char* const* local_dir_ptr; + for(local_dir_ptr = local_dirs; *local_dir_ptr != NULL; ++local_dir_ptr) { + char *container_dir = get_container_work_directory(*local_dir_ptr, user, app_id, + container_id); + if (container_dir == NULL) { + return OUT_OF_MEMORY; + } + char *yarn_sysfs_dir = make_string("%s/%s", container_dir, "sysfs"); + if (mkdir(yarn_sysfs_dir, perms) == 0) { + result = 0; + } + free(yarn_sysfs_dir); + free(container_dir); + } + return result; +} + int launch_docker_container_as_user(const char * user, const char *app_id, const char *container_id, const char *work_dir, const char *script_name, const char *cred_file, @@ -1593,6 +1614,14 @@ int launch_docker_container_as_user(const char * user, const char *app_id, goto cleanup; } + exit_code = create_yarn_sysfs(user, app_id, container_id, work_dir, local_dirs); + if (exit_code != 0) { + fprintf(ERRORFILE, "Could not create user yarn sysfs directory"); + fflush(ERRORFILE); + exit(-1); + goto cleanup; + } + docker_command = construct_docker_command(command_file); docker_binary = get_docker_binary(&CFG); @@ -2546,6 +2575,47 @@ struct configuration* get_cfg() { return &CFG; } +int sync_yarn_sysfs(char* const* local_dir, const char *user, const char *app_id, const char *container_id) { + int result = OUT_OF_MEMORY; + char *src = NULL; + char *dest = NULL; + char* const* local_dir_ptr; + for(local_dir_ptr = local_dir; *local_dir_ptr != NULL; ++local_dir_ptr) { + char *container_dir = make_string("%s/usercache/%s/appcache/%s/%s", *local_dir_ptr, user, app_id, container_id); + char *nm_private_container_dir = make_string("%s/nmPrivate/%s/%s", *local_dir_ptr, app_id, container_id); + + int check = check_nm_local_dir(nm_uid, nm_private_container_dir); + if (check != 0) { + free(container_dir); + free(nm_private_container_dir); + continue; + } + if (container_dir == NULL) { + return OUT_OF_MEMORY; + } + src = make_string("%s/%s", nm_private_container_dir, "service.json"); + dest = make_string("%s/%s", container_dir, "sysfs/service.json"); + // open up the spec file + int spec_file_size = open_file_as_nm(src); + if (spec_file_size == -1) { + continue; + } + + delete_path(dest, 0); + if (copy_file(spec_file_size, src, dest, S_IRWXU | S_IRGRP | S_IXGRP) == 0) { + result = 0; + } + // continue on to create other work directories + free(container_dir); + free(src); + free(dest); + if (result == 0) { + break; + } + } + return result; +} + /** * Flatten docker launch command */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h index 002f85f..b8c7115 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h @@ -32,7 +32,8 @@ enum command { SIGNAL_CONTAINER = 2, DELETE_AS_USER = 3, LAUNCH_DOCKER_CONTAINER = 4, - LIST_AS_USER = 5 + LIST_AS_USER = 5, + SYNC_YARN_SYSFS = 6 }; enum operations { @@ -49,7 +50,8 @@ enum operations { RUN_DOCKER = 11, RUN_AS_USER_LIST = 12, REMOVE_DOCKER_CONTAINER = 13, - INSPECT_DOCKER_CONTAINER = 14 + INSPECT_DOCKER_CONTAINER = 14, + RUN_AS_USER_SYNC_YARN_SYSFS = 15 }; #define NM_GROUP_KEY "yarn.nodemanager.linux-container-executor.group" @@ -274,6 +276,11 @@ int run_docker(const char *command_file); int exec_docker_command(char *docker_command, char **argv, int argc, int optind); +/** + * Sync YARN SysFS + */ +int sync_yarn_sysfs(char* const* local_dirs, const char *user, const char *app_id, const char *container_id); + /* * Compile the regex_str and determine if the input string matches. * Return 0 on match, 1 of non-match. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c index 93691f9..4ca246b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c @@ -100,8 +100,9 @@ static void display_usage(FILE *stream) { fprintf(stream, " signal container: %2d container-pid signal\n" " delete as user: %2d relative-path\n" - " list as user: %2d relative-path\n", - SIGNAL_CONTAINER, DELETE_AS_USER, LIST_AS_USER); + " list as user: %2d relative-path\n" + " sync yarn sysfs: %2d app-id container-id nm-local-dirs\n", + SIGNAL_CONTAINER, DELETE_AS_USER, LIST_AS_USER, SYNC_YARN_SYSFS); } /* Sets up log files for normal/error logging */ @@ -547,6 +548,12 @@ static int validate_run_as_user_commands(int argc, char **argv, int *operation) cmd_input.target_dir = argv[optind++]; *operation = RUN_AS_USER_LIST; return 0; + case SYNC_YARN_SYSFS: + cmd_input.app_id = argv[optind++]; + cmd_input.container_id = argv[optind++]; + cmd_input.local_dirs = argv[optind++]; + *operation = RUN_AS_USER_SYNC_YARN_SYSFS; + return 0; default: fprintf(ERRORFILE, "Invalid command %d not supported.",command); fflush(ERRORFILE); @@ -698,6 +705,10 @@ int main(int argc, char **argv) { exit_code = list_as_user(cmd_input.target_dir); break; + case RUN_AS_USER_SYNC_YARN_SYSFS: + exit_code = sync_yarn_sysfs(split(cmd_input.local_dirs), cmd_input.run_as_user_name, + cmd_input.app_id, cmd_input.container_id); + break; } flush_and_close_log_files(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c index 5607823..c475d84 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c @@ -476,6 +476,68 @@ void test_is_feature_enabled() { free_configuration(&exec_cfg); } +void test_yarn_sysfs() { + char *app_id = "app-1"; + char *container_id = "container-1"; + // Test create sysfs without container. + int result = create_yarn_sysfs(username, app_id, container_id, "work", local_dirs); + if (result == 0) { + printf("Should not be able to create yarn sysfs without container directories.\n"); + exit(1); + } + + result = sync_yarn_sysfs(local_dirs, username, app_id, container_id); + if (result == 0) { + printf("sync_yarn_sysfs failed.\n"); + exit(1); + } + + // Create container directories and init service.json + char* const* local_dir_ptr; + for (local_dir_ptr = local_dirs; *local_dir_ptr != 0; ++local_dir_ptr) { + char *user_dir = make_string("%s/usercache/%s", *local_dir_ptr, username); + if (mkdirs(user_dir, 0750) != 0) { + printf("Can not make user directories: %s\n", user_dir); + exit(1); + } + free(user_dir); + char *app_dir = make_string("%s/usercache/%s/appcache/%s/%s", *local_dir_ptr, username, app_id); + if (mkdirs(app_dir, 0750) != 0) { + printf("Can not make app directories: %s\n", app_dir); + exit(1); + } + free(app_dir); + char *container_dir = make_string("%s/usercache/%s/appcache/%s/%s", *local_dir_ptr, username, app_id, container_id); + if (mkdirs(container_dir, 0750) != 0) { + printf("Can not make container directories: %s\n", container_dir); + exit(1); + } + char *nm_dir = make_string("%s/nmPrivate/%s/%s", *local_dir_ptr, app_id, container_id); + if (mkdirs(nm_dir, 0750) != 0) { + printf("Can not make nmPrivate directories: %s\n", nm_dir); + exit(1); + } + char *sysfs_path = make_string("%s/%s", nm_dir, "service.json"); + FILE *file = fopen(sysfs_path, "w"); + fprintf(file, "{}\n"); + fclose(file); + free(nm_dir); + } + + // Test create sysfs with containers + result = create_yarn_sysfs(username, app_id, container_id, "work", local_dirs); + if (result != 0) { + printf("Can not create yarn sysfs.\n"); + exit(1); + } + + result = sync_yarn_sysfs(local_dirs, username, app_id, container_id); + if (result != 0) { + printf("sync_yarn_sysfs failed.\n"); + exit(1); + } +} + void test_delete_user() { printf("\nTesting delete_user\n"); char* app_dir = get_app_directory(TEST_ROOT "/local-1", yarn_username, "app_3"); @@ -1326,6 +1388,9 @@ int main(int argc, char **argv) { printf("\nTesting is_feature_enabled()\n"); test_is_feature_enabled(); + printf("\nTesting yarn sysfs\n"); + test_yarn_sysfs(); + test_check_user(0); #ifdef __APPLE__ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java index 8aee532..7e4e849 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java @@ -40,6 +40,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEventType; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationException; import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorImpl.ProcessTreeInfo; import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerLivenessContext; import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerReapContext; @@ -120,6 +121,11 @@ public boolean isContainerAlive(ContainerLivenessContext ctx) throws IOException { return true; } + @Override + public void updateYarnSysFS(Context ctx, String user, String appId, + String containerId, String spec) + throws IOException, PrivilegedOperationException { + } } private static class MockContainerEventHandler implements diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md index 447155c..2cf720d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md @@ -351,6 +351,7 @@ environment variables in the application's environment: | `YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER` | Controls whether the Docker container is a privileged container. In order to use privileged containers, the yarn.nodemanager.runtime.linux.docker.privileged-containers.allowed property must be set to true, and the application owner must appear in the value of the yarn.nodemanager.runtime.linux.docker.privileged-containers.acl property. If this environment variable is set to true, a privileged Docker container will be used if allowed. No other value is allowed, so the environment variable should be left unset rather than setting it to false. | | `YARN_CONTAINER_RUNTIME_DOCKER_MOUNTS` | Adds additional volume mounts to the Docker container. The value of the environment variable should be a comma-separated list of mounts. All such mounts must be given as `source:dest[:mode]` and the mode must be "ro" (read-only) or "rw" (read-write) to specify the type of access being requested. If neither is specified, read-write will be assumed. The mode may include a bind propagation option. In that case, the mode should either be of the form `[option]`, `rw+[option]`, or `ro+[option]`. Valid bind propagation options are shared, rshared, slave, rslave, private, and rprivate. The requested mounts will be validated by container-executor based on the values set in container-executor.cfg for `docker.allowed.ro-mounts` and `docker.allowed.rw-mounts`. | | `YARN_CONTAINER_RUNTIME_DOCKER_DELAYED_REMOVAL` | Allows a user to request delayed deletion of the Docker container on a per container basis. If true, Docker containers will not be removed until the duration defined by yarn.nodemanager.delete.debug-delay-sec has elapsed. Administrators can disable this feature through the yarn-site property yarn.nodemanager.runtime.linux.docker.delayed-removal.allowed. This feature is disabled by default. When this feature is disabled or set to false, the container will be removed as soon as it exits. | +| `YARN_CONTAINER_RUNTIME_YARN_SYSFS` | Enable mounting of container working directory sysfs sub-directory into Docker container /hadoop/yarn/sysfs. This is useful for populating cluster information into container. | The first two are required. The remainder can be set as needed. While controlling the container type through environment variables is somewhat less @@ -717,3 +718,17 @@ In yarn-env.sh, define: ``` export YARN_CONTAINER_RUNTIME_DOCKER_RUN_OVERRIDE_DISABLE=true ``` + +Docker Container YARN SysFS Support +----------------------------------- + +YARN SysFS is a pseudo file system provided by the YARN framework that +exports information about clustering information to Docker container. +Cluster information is exported to /hadoop/yarn/sysfs path. This +API allows application developer to obtain clustering information +without external service dependencies. Custom application master can +populate cluster information by calling node manager REST API. +YARN service framework automatically populates cluster information +to /hadoop/yarn/sysfs/service.json. For more information about +YARN service, see: [YARN Service](./yarn-service/Overview.html). +