diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java index 1f9686a..092666d 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java @@ -52,6 +52,7 @@ import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.YarnException; import org.apache.hadoop.yarn.api.ApplicationConstants; +import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; import org.apache.hadoop.yarn.service.AbstractService; /** @@ -280,7 +281,7 @@ private void runSubtask(org.apache.hadoop.mapred.Task task, // Use the AM's local dir env to generate the intermediate step // output files String[] localSysDirs = StringUtils.getTrimmedStrings( - System.getenv(ApplicationConstants.LOCAL_DIR_ENV)); + System.getenv(Environment.LOCAL_DIR.name())); conf.setStrings(MRConfig.LOCAL_DIR, localSysDirs); LOG.info(MRConfig.LOCAL_DIR + " for uber task: " + conf.get(MRConfig.LOCAL_DIR)); diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java index 1052324..7cba10e 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java @@ -111,8 +111,6 @@ public static void setVMEnv(Map environment, MRJobConfig.STDERR_LOGFILE_ENV, getTaskLogFile(TaskLog.LogName.STDERR) ); - environment.put(MRJobConfig.APPLICATION_ATTEMPT_ID_ENV, - conf.get(MRJobConfig.APPLICATION_ATTEMPT_ID).toString()); } private static String getChildJavaOpts(JobConf jobConf, boolean isMapTask) { diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java index 33ff626..551c312 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java @@ -59,6 +59,9 @@ import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler; import org.apache.hadoop.yarn.api.ApplicationConstants; +import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.log4j.LogManager; /** @@ -215,7 +218,7 @@ public Object run() throws Exception { */ private static void configureLocalDirs(Task task, JobConf job) throws IOException { String[] localSysDirs = StringUtils.getTrimmedStrings( - System.getenv(ApplicationConstants.LOCAL_DIR_ENV)); + System.getenv(Environment.LOCAL_DIR.name())); job.setStrings(MRConfig.LOCAL_DIR, localSysDirs); LOG.info(MRConfig.LOCAL_DIR + " for child: " + job.get(MRConfig.LOCAL_DIR)); LocalDirAllocator lDirAlloc = new LocalDirAllocator(MRConfig.LOCAL_DIR); @@ -258,12 +261,13 @@ private static JobConf configureTask(Task task, Credentials credentials, // set job classloader if configured MRApps.setJobClassLoader(job); - String appAttemptIdEnv = System - .getenv(MRJobConfig.APPLICATION_ATTEMPT_ID_ENV); + ApplicationAttemptId appAttemptIdEnv = + ConverterUtils.toContainerId( + System.getenv(Environment.CONTAINER_ID.name())) + .getApplicationAttemptId(); LOG.debug("APPLICATION_ATTEMPT_ID: " + appAttemptIdEnv); // Set it in conf, so as to be able to be used the the OutputCommitter. - job.setInt(MRJobConfig.APPLICATION_ATTEMPT_ID, Integer - .parseInt(appAttemptIdEnv)); + job.setInt(MRJobConfig.APPLICATION_ATTEMPT_ID, appAttemptIdEnv.getAttemptId()); // set tcp nodelay job.setBoolean("ipc.client.tcpnodelay", true); diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java index 45d6e9e..eace269 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java @@ -116,6 +116,7 @@ import org.apache.hadoop.yarn.YarnException; import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler; import org.apache.hadoop.yarn.api.ApplicationConstants; +import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -1270,22 +1271,22 @@ public static void main(String[] args) { try { Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler()); String containerIdStr = - System.getenv(ApplicationConstants.AM_CONTAINER_ID_ENV); - String nodeHostString = System.getenv(ApplicationConstants.NM_HOST_ENV); - String nodePortString = System.getenv(ApplicationConstants.NM_PORT_ENV); + System.getenv(Environment.CONTAINER_ID.name()); + String nodeHostString = System.getenv(Environment.NM_HOST.name()); + String nodePortString = System.getenv(Environment.NM_PORT.name()); String nodeHttpPortString = - System.getenv(ApplicationConstants.NM_HTTP_PORT_ENV); + System.getenv(Environment.NM_HTTP_PORT.name()); String appSubmitTimeStr = System.getenv(ApplicationConstants.APP_SUBMIT_TIME_ENV); String maxAppAttempts = System.getenv(ApplicationConstants.MAX_APP_ATTEMPTS_ENV); validateInputParam(containerIdStr, - ApplicationConstants.AM_CONTAINER_ID_ENV); - validateInputParam(nodeHostString, ApplicationConstants.NM_HOST_ENV); - validateInputParam(nodePortString, ApplicationConstants.NM_PORT_ENV); + Environment.CONTAINER_ID.name()); + validateInputParam(nodeHostString, Environment.NM_HOST.name()); + validateInputParam(nodePortString, Environment.NM_PORT.name()); validateInputParam(nodeHttpPortString, - ApplicationConstants.NM_HTTP_PORT_ENV); + Environment.NM_HTTP_PORT.name()); validateInputParam(appSubmitTimeStr, ApplicationConstants.APP_SUBMIT_TIME_ENV); validateInputParam(maxAppAttempts, diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java index 6e399ee..2f531b0 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java @@ -569,8 +569,6 @@ public static final String STDOUT_LOGFILE_ENV = "STDOUT_LOGFILE_ENV"; public static final String STDERR_LOGFILE_ENV = "STDERR_LOGFILE_ENV"; - public static final String APPLICATION_ATTEMPT_ID_ENV = "APPLICATION_ATTEMPT_ID_ENV"; - // This should be the directory where splits file gets localized on the node // running ApplicationMaster. public static final String JOB_SUBMIT_DIR = "jobSubmitDir"; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java index 02b15c4..273a863 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java @@ -38,30 +38,6 @@ "AppClientSecretEnv"; /** - * The environment variable for CONTAINER_ID. Set in AppMaster environment - * only - */ - public static final String AM_CONTAINER_ID_ENV = "AM_CONTAINER_ID"; - - /** - * The environment variable for the NM_HOST. Set in the AppMaster environment - * only - */ - public static final String NM_HOST_ENV = "NM_HOST"; - - /** - * The environment variable for the NM_PORT. Set in the AppMaster environment - * only - */ - public static final String NM_PORT_ENV = "NM_PORT"; - - /** - * The environment variable for the NM_HTTP_PORT. Set in the AppMaster environment - * only - */ - public static final String NM_HTTP_PORT_ENV = "NM_HTTP_PORT"; - - /** * The environment variable for APP_SUBMIT_TIME. Set in AppMaster environment * only */ @@ -70,8 +46,6 @@ public static final String CONTAINER_TOKEN_FILE_ENV_NAME = UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION; - public static final String LOCAL_DIR_ENV = "YARN_LOCAL_DIRS"; - /** * The environmental variable for APPLICATION_WEB_PROXY_BASE. Set in * ApplicationMaster's environment only. This states that for all non-relative @@ -177,7 +151,37 @@ /** * $HADOOP_YARN_HOME */ - HADOOP_YARN_HOME("HADOOP_YARN_HOME"); + HADOOP_YARN_HOME("HADOOP_YARN_HOME"), + + /** + * $CONTAINER_ID + * Final, exported by NodeManager and non-modifiable by users. + */ + CONTAINER_ID("CONTAINER_ID"), + + /** + * $NM_HOST + * Final, exported by NodeManager and non-modifiable by users. + */ + NM_HOST("NM_HOST"), + + /** + * $NM_HTTP_PORT + * Final, exported by NodeManager and non-modifiable by users. + */ + NM_HTTP_PORT("NM_HTTP_PORT"), + + /** + * $NM_PORT + * Final, exported by NodeManager and non-modifiable by users. + */ + NM_PORT("NM_PORT"), + + /** + * $LOCAL_DIR + * Final, exported by NodeManager and non-modifiable by users. + */ + LOCAL_DIR("LOCAL_DIR"); private final String variable; private Environment(String variable) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java index 965a8aa..d4abc50 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java @@ -45,6 +45,7 @@ import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.yarn.api.AMRMProtocol; import org.apache.hadoop.yarn.api.ApplicationConstants; +import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; import org.apache.hadoop.yarn.api.ContainerExitStatus; import org.apache.hadoop.yarn.api.ContainerManager; @@ -320,7 +321,7 @@ public boolean init(String[] args) throws ParseException, IOException { Map envs = System.getenv(); - if (!envs.containsKey(ApplicationConstants.AM_CONTAINER_ID_ENV)) { + if (!envs.containsKey(Environment.CONTAINER_ID.name())) { if (cliParser.hasOption("app_attempt_id")) { String appIdStr = cliParser.getOptionValue("app_attempt_id", ""); appAttemptID = ConverterUtils.toApplicationAttemptId(appIdStr); @@ -330,7 +331,7 @@ public boolean init(String[] args) throws ParseException, IOException { } } else { ContainerId containerId = ConverterUtils.toContainerId(envs - .get(ApplicationConstants.AM_CONTAINER_ID_ENV)); + .get(Environment.CONTAINER_ID.name())); appAttemptID = containerId.getApplicationAttemptId(); } @@ -338,16 +339,16 @@ public boolean init(String[] args) throws ParseException, IOException { throw new RuntimeException(ApplicationConstants.APP_SUBMIT_TIME_ENV + " not set in the environment"); } - if (!envs.containsKey(ApplicationConstants.NM_HOST_ENV)) { - throw new RuntimeException(ApplicationConstants.NM_HOST_ENV + if (!envs.containsKey(Environment.NM_HOST.name())) { + throw new RuntimeException(Environment.NM_HOST.name() + " not set in the environment"); } - if (!envs.containsKey(ApplicationConstants.NM_HTTP_PORT_ENV)) { - throw new RuntimeException(ApplicationConstants.NM_HTTP_PORT_ENV + if (!envs.containsKey(Environment.NM_HTTP_PORT.name())) { + throw new RuntimeException(Environment.NM_HTTP_PORT + " not set in the environment"); } - if (!envs.containsKey(ApplicationConstants.NM_PORT_ENV)) { - throw new RuntimeException(ApplicationConstants.NM_PORT_ENV + if (!envs.containsKey(Environment.NM_PORT.name())) { + throw new RuntimeException(Environment.NM_PORT.name() + " not set in the environment"); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java index 605dc3b..ea378d2 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java @@ -37,6 +37,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.ApplicationConstants; +import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -190,10 +191,10 @@ public void launchAM(ApplicationAttemptId attemptId) throws IOException { containerId.setId(0); String hostname = InetAddress.getLocalHost().getHostName(); - envAMList.add(ApplicationConstants.AM_CONTAINER_ID_ENV + "=" + containerId); - envAMList.add(ApplicationConstants.NM_HOST_ENV + "=" + hostname); - envAMList.add(ApplicationConstants.NM_HTTP_PORT_ENV + "=0"); - envAMList.add(ApplicationConstants.NM_PORT_ENV + "=0"); + envAMList.add(Environment.CONTAINER_ID.name() + "=" + containerId); + envAMList.add(Environment.NM_HOST.name() + "=" + hostname); + envAMList.add(Environment.NM_HTTP_PORT.name() + "=0"); + envAMList.add(Environment.NM_PORT.name() + "=0"); envAMList.add(ApplicationConstants.APP_SUBMIT_TIME_ENV + "=" + System.currentTimeMillis()); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java index e42d74d..89032f2 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java @@ -113,13 +113,13 @@ public int launchContainer(Container container, List localDirs, List logDirs) throws IOException { FsPermission dirPerm = new FsPermission(APPDIR_PERM); - ContainerId containerId = container.getContainerID(); + ContainerId containerId = container.getContainer().getId(); // create container dirs on all disks String containerIdStr = ConverterUtils.toString(containerId); String appIdStr = ConverterUtils.toString( - container.getContainerID().getApplicationAttemptId(). + containerId.getApplicationAttemptId(). getApplicationId()); for (String sLocalDir : localDirs) { Path usersdir = new Path(sLocalDir, ContainerLocalizer.USERCACHE); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java index e0a3582..0c6b502 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java @@ -216,11 +216,11 @@ public int launchContainer(Container container, String user, String appId, Path containerWorkDir, List localDirs, List logDirs) throws IOException { - ContainerId containerId = container.getContainerID(); + ContainerId containerId = container.getContainer().getId(); String containerIdStr = ConverterUtils.toString(containerId); resourcesHandler.preExecute(containerId, - container.getResource()); + container.getContainer().getResource()); String resourcesOptions = resourcesHandler.getResourcesOption( containerId); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationContainerInitEvent.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationContainerInitEvent.java index 88be9d1..b74be63 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationContainerInitEvent.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationContainerInitEvent.java @@ -36,8 +36,8 @@ final Container container; public ApplicationContainerInitEvent(Container container) { - super(container.getContainerID().getApplicationAttemptId().getApplicationId(), - ApplicationEventType.INIT_CONTAINER); + super(container.getContainer().getId().getApplicationAttemptId() + .getApplicationId(), ApplicationEventType.INIT_CONTAINER); this.container = container; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java index e48e2fa..4e4cf0a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java @@ -274,14 +274,14 @@ public void transition(ApplicationImpl app, ApplicationEvent event) { ApplicationContainerInitEvent initEvent = (ApplicationContainerInitEvent) event; Container container = initEvent.getContainer(); - app.containers.put(container.getContainerID(), container); - LOG.info("Adding " + container.getContainerID() + app.containers.put(container.getContainer().getId(), container); + LOG.info("Adding " + container.getContainer().getId() + " to application " + app.toString()); switch (app.getApplicationState()) { case RUNNING: app.dispatcher.getEventHandler().handle(new ContainerInitEvent( - container.getContainerID())); + container.getContainer().getId())); break; case INITING: case NEW: @@ -302,7 +302,7 @@ public void transition(ApplicationImpl app, ApplicationEvent event) { // Start all the containers waiting for ApplicationInit for (Container container : app.containers.values()) { app.dispatcher.getEventHandler().handle(new ContainerInitEvent( - container.getContainerID())); + container.getContainer().getId())); } } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java index a43e1b7..c975101 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java @@ -25,12 +25,11 @@ import org.apache.hadoop.security.Credentials; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerStatus; -import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.event.EventHandler; public interface Container extends EventHandler { - org.apache.hadoop.yarn.api.records.ContainerId getContainerID(); + org.apache.hadoop.yarn.api.records.Container getContainer(); String getUser(); @@ -46,5 +45,4 @@ String toString(); - Resource getResource(); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java index 961c454..1195763 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java @@ -42,7 +42,6 @@ import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; -import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.EventHandler; @@ -313,16 +312,6 @@ public ContainerImpl(Configuration conf, } @Override - public ContainerId getContainerID() { - this.readLock.lock(); - try { - return this.container.getId(); - } finally { - this.readLock.unlock(); - } - } - - @Override public String getUser() { this.readLock.lock(); try { @@ -385,10 +374,10 @@ public ContainerStatus cloneAndGetContainerStatus() { } @Override - public Resource getResource() { + public org.apache.hadoop.yarn.api.records.Container getContainer() { this.readLock.lock(); try { - return this.container.getResource(); + return this.container; } finally { this.readLock.unlock(); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java index d48a4b7..dfa0a52 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java @@ -118,7 +118,7 @@ public Integer call() { final ContainerLaunchContext launchContext = container.getLaunchContext(); final Map> localResources = container.getLocalizedResources(); - ContainerId containerID = container.getContainerID(); + ContainerId containerID = container.getContainer().getId(); String containerIdStr = ConverterUtils.toString(containerID); final String user = launchContext.getUser(); final List command = launchContext.getCommands(); @@ -299,7 +299,7 @@ public Integer call() { * @throws IOException */ public void cleanupContainer() throws IOException { - ContainerId containerId = container.getContainerID(); + ContainerId containerId = container.getContainer().getId(); String containerIdStr = ConverterUtils.toString(containerId); LOG.info("Cleaning up container " + containerIdStr); @@ -370,7 +370,7 @@ public void cleanupContainer() throws IOException { */ private String getContainerPid(Path pidFilePath) throws Exception { String containerIdStr = - ConverterUtils.toString(container.getContainerID()); + ConverterUtils.toString(container.getContainer().getId()); String processId = null; LOG.debug("Accessing pid for container " + containerIdStr + " from pid file " + pidFilePath); @@ -547,6 +547,20 @@ public void sanitizeEnv(Map environment, * Non-modifiable environment variables */ + environment.put(Environment.CONTAINER_ID.name(), container + .getContainer().getId().toString()); + + environment.put(Environment.NM_PORT.name(), + String.valueOf(container.getContainer().getNodeId().getPort())); + + environment.put(Environment.NM_HOST.name(), container.getContainer() + .getNodeId().getHost()); + + environment.put(Environment.NM_HTTP_PORT.name(), container.getContainer() + .getNodeHttpAddress().split(":")[1]); + + environment.put(Environment.LOCAL_DIR.name(), StringUtils.join(",", appDirs)); + putEnvIfNotNull(environment, Environment.USER.name(), container.getUser()); putEnvIfNotNull(environment, @@ -566,11 +580,6 @@ public void sanitizeEnv(Map environment, Environment.HADOOP_CONF_DIR.name(), System.getenv(Environment.HADOOP_CONF_DIR.name()) ); - - putEnvIfNotNull(environment, - ApplicationConstants.LOCAL_DIR_ENV, - StringUtils.join(",", appDirs) - ); if (!Shell.WINDOWS) { environment.put("JVM_PID", "$$"); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java index 1e3c18b..22cffb9 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java @@ -111,7 +111,7 @@ public void stop() { public void handle(ContainersLauncherEvent event) { // TODO: ContainersLauncher launches containers one by one!! Container container = event.getContainer(); - ContainerId containerId = container.getContainerID(); + ContainerId containerId = container.getContainer().getId(); switch (event.getType()) { case LAUNCH_CONTAINER: Application app = diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java index 7b9873a..85ff962 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java @@ -356,13 +356,15 @@ private void handleInitContainerResources( ContainerLocalizationRequestEvent rsrcReqs) { Container c = rsrcReqs.getContainer(); LocalizerContext ctxt = new LocalizerContext( - c.getUser(), c.getContainerID(), c.getCredentials()); + c.getUser(), c.getContainer().getId(), c.getCredentials()); Map> rsrcs = rsrcReqs.getRequestedResources(); for (Map.Entry> e : rsrcs.entrySet()) { - LocalResourcesTracker tracker = getLocalResourcesTracker(e.getKey(), c.getUser(), - c.getContainerID().getApplicationAttemptId().getApplicationId()); + LocalResourcesTracker tracker = + getLocalResourcesTracker(e.getKey(), c.getUser(), + c.getContainer().getId().getApplicationAttemptId() + .getApplicationId()); for (LocalResourceRequest req : e.getValue()) { tracker.handle(new ResourceRequestEvent(req, e.getKey(), ctxt)); } @@ -391,19 +393,21 @@ private void handleCleanupContainerResources( for (Map.Entry> e : rsrcs.entrySet()) { LocalResourcesTracker tracker = getLocalResourcesTracker(e.getKey(), c.getUser(), - c.getContainerID().getApplicationAttemptId().getApplicationId()); + c.getContainer().getId().getApplicationAttemptId() + .getApplicationId()); for (LocalResourceRequest req : e.getValue()) { - tracker.handle(new ResourceReleaseEvent(req, c.getContainerID())); + tracker.handle(new ResourceReleaseEvent(req, + c.getContainer().getId())); } } - String locId = ConverterUtils.toString(c.getContainerID()); + String locId = ConverterUtils.toString(c.getContainer().getId()); localizerTracker.cleanupPrivLocalizers(locId); // Delete the container directories String userName = c.getUser(); String containerIDStr = c.toString(); String appIDStr = ConverterUtils.toString( - c.getContainerID().getApplicationAttemptId().getApplicationId()); + c.getContainer().getId().getApplicationAttemptId().getApplicationId()); for (String localDir : dirsHandler.getLocalDirs()) { // Delete the user-owned container-dir @@ -422,8 +426,9 @@ private void handleCleanupContainerResources( delService.delete(null, containerSysDir, new Path[] {}); } - dispatcher.getEventHandler().handle(new ContainerEvent(c.getContainerID(), - ContainerEventType.CONTAINER_RESOURCES_CLEANEDUP)); + dispatcher.getEventHandler().handle( + new ContainerEvent(c.getContainer().getId(), + ContainerEventType.CONTAINER_RESOURCES_CLEANEDUP)); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java index 712d8ff..379d998 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java @@ -60,7 +60,7 @@ public ContainerInfo(final Context nmContext, final Container container) { public ContainerInfo(final Context nmContext, final Container container, String requestUri, String pathPrefix) { - this.id = container.getContainerID().toString(); + this.id = container.getContainer().getId().toString(); this.nodeId = nmContext.getNodeId().toString(); ContainerStatus containerData = container.cloneAndGetContainerStatus(); this.exitCode = containerData.getExitStatus(); @@ -74,7 +74,7 @@ public ContainerInfo(final Context nmContext, final Container container, } this.user = container.getUser(); - Resource res = container.getResource(); + Resource res = container.getContainer().getResource(); if (res != null) { this.totalMemoryNeededMB = res.getMemory(); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java index d848405..3284634 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java @@ -88,10 +88,10 @@ public void handle(LocalizationEvent event) { .getRequestedResources().values()) { for (LocalResourceRequest req : rc) { LOG.info("DEBUG: " + req + ":" - + rsrcReqs.getContainer().getContainerID()); + + rsrcReqs.getContainer().getContainer().getId()); dispatcher.getEventHandler().handle( new ContainerResourceLocalizedEvent(rsrcReqs.getContainer() - .getContainerID(), req, new Path("file:///local" + .getContainer().getId(), req, new Path("file:///local" + req.getPath().toUri().getPath()))); } } @@ -101,7 +101,7 @@ public void handle(LocalizationEvent event) { ((ContainerLocalizationEvent) event).getContainer(); // TODO: delete the container dir this.dispatcher.getEventHandler().handle( - new ContainerEvent(container.getContainerID(), + new ContainerEvent(container.getContainer().getId(), ContainerEventType.CONTAINER_RESOURCES_CLEANEDUP)); break; case DESTROY_APPLICATION_RESOURCES: @@ -130,7 +130,7 @@ protected ContainersLauncher createContainersLauncher(Context context, @Override public void handle(ContainersLauncherEvent event) { Container container = event.getContainer(); - ContainerId containerId = container.getContainerID(); + ContainerId containerId = container.getContainer().getId(); switch (event.getType()) { case LAUNCH_CONTAINER: dispatcher.getEventHandler().handle( diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java index 02c2001..805f294 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java @@ -186,7 +186,10 @@ private int runAndBlock(ContainerId cId, String ... cmd) throws IOException { ContainerLaunchContext context = mock(ContainerLaunchContext.class); HashMap env = new HashMap(); - when(container.getContainerID()).thenReturn(cId); + org.apache.hadoop.yarn.api.records.Container containerAPI = + mock(org.apache.hadoop.yarn.api.records.Container.class); + when(container.getContainer()).thenReturn(containerAPI); + when(container.getContainer().getId()).thenReturn(cId); when(container.getLaunchContext()).thenReturn(context); when(context.getEnvironment()).thenReturn(env); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java index e6977c8..118d58a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java @@ -107,7 +107,10 @@ public void testContainerLaunch() throws IOException { ContainerLaunchContext context = mock(ContainerLaunchContext.class); HashMap env = new HashMap(); - when(container.getContainerID()).thenReturn(cId); + org.apache.hadoop.yarn.api.records.Container containerAPI = + mock(org.apache.hadoop.yarn.api.records.Container.class); + when(container.getContainer()).thenReturn(containerAPI); + when(container.getContainer().getId()).thenReturn(cId); when(container.getLaunchContext()).thenReturn(context); when(cId.toString()).thenReturn(containerId); @@ -225,7 +228,10 @@ public void testContainerLaunchError() throws IOException { ContainerLaunchContext context = mock(ContainerLaunchContext.class); HashMap env = new HashMap(); - when(container.getContainerID()).thenReturn(cId); + org.apache.hadoop.yarn.api.records.Container containerAPI = + mock(org.apache.hadoop.yarn.api.records.Container.class); + when(container.getContainer()).thenReturn(containerAPI); + when(container.getContainer().getId()).thenReturn(cId); when(container.getLaunchContext()).thenReturn(context); when(cId.toString()).thenReturn(containerId); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java index df170c7..1792988 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java @@ -54,6 +54,7 @@ import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; +import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.URL; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -178,6 +179,9 @@ private void startContainers(NodeManager nm) throws IOException { Container mockContainer = mock(Container.class); when(mockContainer.getId()).thenReturn(cId); + NodeId nodeId = BuilderUtils.newNodeId("localhost", 1234); + when(mockContainer.getNodeId()).thenReturn(nodeId); + when(mockContainer.getNodeHttpAddress()).thenReturn("localhost:12345"); containerLaunchContext.setUser(user); URL localResourceUri = diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java index b5a4d8c..981ab39 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java @@ -150,6 +150,10 @@ public void testContainerSetup() throws IOException, InterruptedException { when(mockContainer.getId()).thenReturn(cId); when(mockContainer.getResource()).thenReturn( BuilderUtils.newResource(512, 1)); + when(mockContainer.getNodeId()).thenReturn(context.getNodeId()); + when(mockContainer.getNodeId()).thenReturn(context.getNodeId()); + when(mockContainer.getNodeHttpAddress()).thenReturn( + context.getNodeId().getHost() + ":12345"); StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class); @@ -245,6 +249,10 @@ public void testContainerLaunchAndStop() throws IOException, when(mockContainer.getId()).thenReturn(cId); when(mockContainer.getResource()).thenReturn( BuilderUtils.newResource(100, 1)); // MB + when(mockContainer.getNodeId()).thenReturn(context.getNodeId()); + when(mockContainer.getNodeHttpAddress()).thenReturn( + context.getNodeId().getHost() + ":12345"); + StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class); startRequest.setContainerLaunchContext(containerLaunchContext); startRequest.setContainer(mockContainer); @@ -352,7 +360,9 @@ private void testContainerLaunchAndExit(int exitCode) throws IOException, Interr when(mockContainer.getId()).thenReturn(cId); when(mockContainer.getResource()).thenReturn( BuilderUtils.newResource(100, 1)); // MB - + when(mockContainer.getNodeId()).thenReturn(context.getNodeId()); + when(mockContainer.getNodeHttpAddress()).thenReturn( + context.getNodeId().getHost() + ":12345"); StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class); startRequest.setContainerLaunchContext(containerLaunchContext); startRequest.setContainer(mockContainer); @@ -444,6 +454,9 @@ public void testLocalFilesCleanup() throws InterruptedException, when(mockContainer.getId()).thenReturn(cId); when(mockContainer.getResource()).thenReturn( BuilderUtils.newResource(100, 1)); + when(mockContainer.getNodeId()).thenReturn(context.getNodeId()); + when(mockContainer.getNodeHttpAddress()).thenReturn( + context.getNodeId().getHost() + ":12345"); // containerLaunchContext.command = new ArrayList(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java index 3dd8d27..1050a9e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java @@ -83,7 +83,7 @@ public void testApplicationInit1() { for (int i = 0; i < wa.containers.size(); i++) { verify(wa.containerBus).handle( argThat(new ContainerInitMatcher(wa.containers.get(i) - .getContainerID()))); + .getContainer().getId()))); } } finally { if (wa != null) @@ -108,7 +108,7 @@ public void testApplicationInit2() { assertEquals(ApplicationState.RUNNING, wa.app.getApplicationState()); verify(wa.containerBus).handle( argThat(new ContainerInitMatcher(wa.containers.get(0) - .getContainerID()))); + .getContainer().getId()))); wa.initContainer(1); wa.initContainer(2); @@ -118,7 +118,7 @@ public void testApplicationInit2() { for (int i = 1; i < wa.containers.size(); i++) { verify(wa.containerBus).handle( argThat(new ContainerInitMatcher(wa.containers.get(i) - .getContainerID()))); + .getContainer().getId()))); } } finally { if (wa != null) @@ -233,7 +233,7 @@ public void testAppFinishedOnRunningContainers() { for (int i = 1; i < wa.containers.size(); i++) { verify(wa.containerBus).handle( argThat(new ContainerKillMatcher(wa.containers.get(i) - .getContainerID()))); + .getContainer().getId()))); } wa.containerFinished(1); @@ -354,7 +354,7 @@ public void testAppFinishedOnIniting() { verify(wa.containerBus).handle( argThat(new ContainerKillMatcher(wa.containers.get(0) - .getContainerID()))); + .getContainer().getId()))); assertEquals(ApplicationState.FINISHING_CONTAINERS_WAIT, wa.app.getApplicationState()); @@ -487,7 +487,7 @@ public void initContainer(int containerNum) { public void containerFinished(int containerNum) { app.handle(new ApplicationContainerFinishedEvent(containers.get( - containerNum).getContainerID())); + containerNum).getContainer().getId())); drainDispatcherEvents(); } @@ -514,7 +514,10 @@ private Container createMockedContainer(ApplicationId appId, int containerId) { BuilderUtils.newApplicationAttemptId(appId, 1); ContainerId cId = BuilderUtils.newContainerId(appAttemptId, containerId); Container c = mock(Container.class); - when(c.getContainerID()).thenReturn(cId); + org.apache.hadoop.yarn.api.records.Container containerAPI = + mock(org.apache.hadoop.yarn.api.records.Container.class); + when(c.getContainer()).thenReturn(containerAPI); + when(c.getContainer().getId()).thenReturn(cId); ContainerLaunchContext launchContext = mock(ContainerLaunchContext.class); when(c.getLaunchContext()).thenReturn(launchContext); when(launchContext.getApplicationACLs()).thenReturn( diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java index 230ce46..c98379c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java @@ -376,7 +376,7 @@ public boolean matches(Object o) { public boolean matches(Object o) { ContainersLauncherEvent evt = (ContainersLauncherEvent) o; return evt.getType() == ContainersLauncherEventType.LAUNCH_CONTAINER - && wcf.cId == evt.getContainer().getContainerID(); + && wcf.cId == evt.getContainer().getContainer().getId(); } }; verify(wc.launcherBus).handle(argThat(matchesLaunchReq)); @@ -639,7 +639,7 @@ public void initContainer() { Path p = new Path(cache, rsrc.getKey()); localPaths.put(p, Arrays.asList(rsrc.getKey())); // rsrc copied to p - c.handle(new ContainerResourceLocalizedEvent(c.getContainerID(), + c.handle(new ContainerResourceLocalizedEvent(c.getContainer().getId(), req, p)); } drainDispatcherEvents(); @@ -662,7 +662,8 @@ public void failLocalizeSpecificResource(String rsrcKey) LocalResource rsrc = localResources.get(rsrcKey); LocalResourceRequest req = new LocalResourceRequest(rsrc); Exception e = new Exception("Fake localization error"); - c.handle(new ContainerResourceFailedEvent(c.getContainerID(), req, e)); + c.handle(new ContainerResourceFailedEvent(c.getContainer() + .getId(), req, e)); drainDispatcherEvents(); } @@ -677,7 +678,7 @@ public void failLocalizeResources(int failRsrcCount) ++counter; LocalResourceRequest req = new LocalResourceRequest(rsrc.getValue()); Exception e = new Exception("Fake localization error"); - c.handle(new ContainerResourceFailedEvent(c.getContainerID(), + c.handle(new ContainerResourceFailedEvent(c.getContainer().getId(), req, e)); } drainDispatcherEvents(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java index edb9abd..16081c6 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java @@ -37,6 +37,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.UnsupportedFileSystemException; import org.apache.hadoop.util.Shell; +import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; @@ -51,14 +52,13 @@ import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; -import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.URL; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode; -import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.Signal; import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor; import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest; import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer; import org.apache.hadoop.yarn.util.BuilderUtils; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.LinuxResourceCalculatorPlugin; @@ -186,14 +186,12 @@ public static void setNewEnvironmentHack(Map newenv) throws Exce * See if environment variable is forwarded using sanitizeEnv. * @throws Exception */ - @Test + @Test (timeout = 5000) public void testContainerEnvVariables() throws Exception { containerManager.start(); - Map envWithDummy = new HashMap(); - envWithDummy.putAll(System.getenv()); - envWithDummy.put(Environment.MALLOC_ARENA_MAX.name(), "99"); - setNewEnvironmentHack(envWithDummy); + ContainerLaunchContext containerLaunchContext = + recordFactory.newRecordInstance(ContainerLaunchContext.class); Container mockContainer = mock(Container.class); // ////// Construct the Container-id @@ -207,6 +205,31 @@ public void testContainerEnvVariables() throws Exception { ContainerId cId = recordFactory.newRecordInstance(ContainerId.class); cId.setApplicationAttemptId(appAttemptId); + when(mockContainer.getId()).thenReturn(cId); + + when(mockContainer.getNodeId()).thenReturn(context.getNodeId()); + when(mockContainer.getNodeHttpAddress()).thenReturn( + context.getNodeId().getHost() + ":12345"); + + Map userSetEnv = new HashMap(); + userSetEnv.put(Environment.CONTAINER_ID.name(), "user_set_container_id"); + userSetEnv.put(Environment.NM_HOST.name(), "user_set_NM_HOST"); + userSetEnv.put(Environment.NM_PORT.name(), "user_set_NM_PORT"); + userSetEnv.put(Environment.NM_HTTP_PORT.name(), "user_set_NM_HTTP_PORT"); + userSetEnv.put(Environment.LOCAL_DIR.name(), "user_set_LOCAL_DIR"); + containerLaunchContext.setUser(user); + containerLaunchContext.setEnvironment(userSetEnv); + + Map envWithDummy = new HashMap(); + envWithDummy.putAll(System.getenv()); + envWithDummy.put(Environment.MALLOC_ARENA_MAX.name(), "99"); + envWithDummy.put(Environment.CONTAINER_ID.name(), cId.toString()); + envWithDummy.put(Environment.NM_HOST.name(), + context.getNodeId().getHost()); + envWithDummy.put(Environment.NM_PORT.name(), + String.valueOf(context.getNodeId().getPort())); + setNewEnvironmentHack(envWithDummy); + String malloc = System.getenv(Environment.MALLOC_ARENA_MAX.name()); File scriptFile = Shell.appendScriptExtension(tmpDir, "scriptFile"); PrintWriter fileWriter = new PrintWriter(scriptFile); @@ -215,12 +238,26 @@ public void testContainerEnvVariables() throws Exception { if (Shell.WINDOWS) { fileWriter.println("@echo " + Environment.MALLOC_ARENA_MAX.$() + "> " + processStartFile); - fileWriter.println("@echo " + cId + ">> " + processStartFile); + fileWriter.println("@echo " + Environment.CONTAINER_ID.$() + ">> " + processStartFile); + fileWriter.println("@echo " + Environment.NM_HOST.$() + ">> " + processStartFile); + fileWriter.println("@echo " + Environment.NM_PORT.$() + ">> " + processStartFile); + fileWriter.println("@echo " + Environment.NM_HTTP_PORT.$() + ">> " + processStartFile); + fileWriter.println("@echo " + Environment.LOCAL_DIR.$() + ">> " + processStartFile); fileWriter.println("@ping -n 100 127.0.0.1 >nul"); } else { fileWriter.write("\numask 0"); // So that start file is readable by the test - fileWriter.write("\necho " + Environment.MALLOC_ARENA_MAX.$() + " > " + - processStartFile); + fileWriter.write("\necho $" + Environment.MALLOC_ARENA_MAX.name() + " > " + + processStartFile); + fileWriter.write("\necho $" + Environment.CONTAINER_ID.name() + " >> " + + processStartFile); + fileWriter.write("\necho $" + Environment.NM_HOST.name() + " >> " + + processStartFile); + fileWriter.write("\necho $" + Environment.NM_PORT.name() + " >> " + + processStartFile); + fileWriter.write("\necho $" + Environment.NM_HTTP_PORT.name() + " >> " + + processStartFile); + fileWriter.write("\necho $" + Environment.LOCAL_DIR.name() + " >> " + + processStartFile); fileWriter.write("\necho $$ >> " + processStartFile); fileWriter.write("\nexec sleep 100"); } @@ -228,13 +265,6 @@ public void testContainerEnvVariables() throws Exception { assert(malloc != null && !"".equals(malloc)); - ContainerLaunchContext containerLaunchContext = - recordFactory.newRecordInstance(ContainerLaunchContext.class); - - when(mockContainer.getId()).thenReturn(cId); - - containerLaunchContext.setUser(user); - // upload the script file so that the container can run it URL resource_alpha = ConverterUtils.getYarnUrlFromPath(localFS @@ -272,9 +302,39 @@ public void testContainerEnvVariables() throws Exception { processStartFile.exists()); // Now verify the contents of the file + List localDirs = dirsHandler.getLocalDirs(); + List appDirs = new ArrayList(localDirs.size()); + for (String localDir : localDirs) { + Path usersdir = new Path(localDir, ContainerLocalizer.USERCACHE); + Path userdir = new Path(usersdir, user); + Path appsdir = new Path(userdir, ContainerLocalizer.APPCACHE); + appDirs.add(new Path(appsdir, appId.toString())); + } BufferedReader reader = new BufferedReader(new FileReader(processStartFile)); Assert.assertEquals(malloc, reader.readLine()); + Assert.assertEquals(cId.toString(), reader.readLine()); + Assert.assertEquals(mockContainer.getNodeId().getHost(), reader.readLine()); + Assert.assertEquals(String.valueOf(mockContainer.getNodeId().getPort()), + reader.readLine()); + Assert.assertEquals( + String.valueOf(mockContainer.getNodeHttpAddress().split(":")[1]), + reader.readLine()); + Assert.assertEquals(StringUtils.join(",", appDirs), reader.readLine()); + + Assert.assertEquals(cId.toString(), containerLaunchContext.getEnvironment() + .get(Environment.CONTAINER_ID.name())); + Assert.assertEquals(mockContainer.getNodeId().getHost(), containerLaunchContext + .getEnvironment().get(Environment.NM_HOST.name())); + Assert.assertEquals(String.valueOf(mockContainer.getNodeId().getPort()), + containerLaunchContext.getEnvironment().get( + Environment.NM_PORT.name())); + Assert.assertEquals( + mockContainer.getNodeHttpAddress().split(":")[1], + containerLaunchContext.getEnvironment().get( + Environment.NM_HTTP_PORT.name())); + Assert.assertEquals(StringUtils.join(",", appDirs), containerLaunchContext + .getEnvironment().get(Environment.LOCAL_DIR.name())); // Get the pid of the process String pid = reader.readLine().trim(); // No more lines @@ -354,6 +414,9 @@ public void testDelayedKill() throws Exception { ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class); when(mockContainer.getId()).thenReturn(cId); + when(mockContainer.getNodeId()).thenReturn(context.getNodeId()); + when(mockContainer.getNodeHttpAddress()).thenReturn( + context.getNodeId().getHost() + ":12345"); containerLaunchContext.setUser(user); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java index 77bde7b..303315c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java @@ -481,7 +481,7 @@ public boolean matches(Object o) { Thread.sleep(1000); dispatcher.await(); String appStr = ConverterUtils.toString(appId); - String ctnrStr = c.getContainerID().toString(); + String ctnrStr = c.getContainer().getId().toString(); ArgumentCaptor tokenPathCaptor = ArgumentCaptor.forClass(Path.class); verify(exec).startLocalizer(tokenPathCaptor.capture(), isA(InetSocketAddress.class), eq("user0"), eq(appStr), eq(ctnrStr), @@ -557,7 +557,7 @@ public boolean matches(Object o) { public boolean matches(Object o) { ContainerEvent evt = (ContainerEvent) o; return evt.getType() == ContainerEventType.RESOURCE_LOCALIZED - && c.getContainerID() == evt.getContainerID(); + && c.getContainer().getId() == evt.getContainerID(); } }; // total 2 resource localzation calls. one for each resource. @@ -709,8 +709,11 @@ private static Container getMockContainer(ApplicationId appId, int id) { ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(appId, 1); ContainerId cId = BuilderUtils.newContainerId(appAttemptId, id); + org.apache.hadoop.yarn.api.records.Container containerAPI = + mock(org.apache.hadoop.yarn.api.records.Container.class); + when(c.getContainer()).thenReturn(containerAPI); when(c.getUser()).thenReturn("user0"); - when(c.getContainerID()).thenReturn(cId); + when(c.getContainer().getId()).thenReturn(cId); Credentials creds = new Credentials(); creds.addToken(new Text("tok" + id), getToken(id)); when(c.getCredentials()).thenReturn(creds); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java index a27b357..ad4a818 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java @@ -213,6 +213,9 @@ public void testContainerKillOnMemoryOverflow() throws IOException, cId.setApplicationAttemptId(appAttemptId); when(mockContainer.getId()).thenReturn(cId); + when(mockContainer.getNodeId()).thenReturn(context.getNodeId()); + when(mockContainer.getNodeHttpAddress()).thenReturn( + context.getNodeId().getHost() + ":12345"); containerLaunchContext.setUser(user); URL resource_alpha = diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java index dbb50bb..f233173 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java @@ -38,6 +38,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState; import org.apache.hadoop.yarn.util.BuilderUtils; +import static org.mockito.Mockito.*; public class MockContainer implements Container { @@ -48,6 +49,7 @@ private final Map> resource = new HashMap>(); private RecordFactory recordFactory; + private org.apache.hadoop.yarn.api.records.Container mockContainer; public MockContainer(ApplicationAttemptId appAttemptId, Dispatcher dispatcher, Configuration conf, String user, @@ -62,6 +64,8 @@ public MockContainer(ApplicationAttemptId appAttemptId, launchContext.setUser(user); this.state = ContainerState.NEW; + mockContainer = mock(org.apache.hadoop.yarn.api.records.Container.class); + when(mockContainer.getId()).thenReturn(id); } public void setState(ContainerState state) { @@ -69,11 +73,6 @@ public void setState(ContainerState state) { } @Override - public ContainerId getContainerID() { - return id; - } - - @Override public String getUser() { return user; } @@ -119,8 +118,7 @@ public void handle(ContainerEvent event) { } @Override - public Resource getResource() { - return null; + public org.apache.hadoop.yarn.api.records.Container getContainer() { + return this.mockContainer; } - } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java index 39764e6..bde9fee 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java @@ -185,16 +185,18 @@ public void testNodeAppsNone() throws JSONException, Exception { app.getUser(), app.getAppId(), 1); Container container2 = new MockContainer(appAttemptId, dispatcher, conf, app.getUser(), app.getAppId(), 2); - nmContext.getContainers().put(container1.getContainerID(), container1); - nmContext.getContainers().put(container2.getContainerID(), container2); + nmContext.getContainers() + .put(container1.getContainer().getId(), container1); + nmContext.getContainers() + .put(container2.getContainer().getId(), container2); - app.getContainers().put(container1.getContainerID(), container1); - app.getContainers().put(container2.getContainerID(), container2); + app.getContainers().put(container1.getContainer().getId(), container1); + app.getContainers().put(container2.getContainer().getId(), container2); HashMap hash = new HashMap(); - hash.put(container1.getContainerID().toString(), container1 - .getContainerID().toString()); - hash.put(container2.getContainerID().toString(), container2 - .getContainerID().toString()); + hash.put(container1.getContainer().getId().toString(), container1 + .getContainer().getId().toString()); + hash.put(container2.getContainer().getId().toString(), container2 + .getContainer().getId().toString()); return hash; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java index 1f678d4..436fefa 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java @@ -186,16 +186,18 @@ public void testNodeContainersNone() throws JSONException, Exception { app.getUser(), app.getAppId(), 1); Container container2 = new MockContainer(appAttemptId, dispatcher, conf, app.getUser(), app.getAppId(), 2); - nmContext.getContainers().put(container1.getContainerID(), container1); - nmContext.getContainers().put(container2.getContainerID(), container2); + nmContext.getContainers() + .put(container1.getContainer().getId(), container1); + nmContext.getContainers() + .put(container2.getContainer().getId(), container2); - app.getContainers().put(container1.getContainerID(), container1); - app.getContainers().put(container2.getContainerID(), container2); + app.getContainers().put(container1.getContainer().getId(), container1); + app.getContainers().put(container2.getContainer().getId(), container2); HashMap hash = new HashMap(); - hash.put(container1.getContainerID().toString(), container1 - .getContainerID().toString()); - hash.put(container2.getContainerID().toString(), container2 - .getContainerID().toString()); + hash.put(container1.getContainer().getId().toString(), container1 + .getContainer().getId().toString()); + hash.put(container2.getContainer().getId().toString(), container2 + .getContainer().getId().toString()); return hash; } @@ -468,7 +470,7 @@ public void verifyNodeContainerInfoGeneric(Container cont, String id, String state, String user, int exitCode, String diagnostics, String nodeId, int totalMemoryNeededMB, String logsLink) throws JSONException, Exception { - WebServicesTestUtils.checkStringMatch("id", cont.getContainerID() + WebServicesTestUtils.checkStringMatch("id", cont.getContainer().getId() .toString(), id); WebServicesTestUtils.checkStringMatch("state", cont.getContainerState() .toString(), state); @@ -481,7 +483,7 @@ public void verifyNodeContainerInfoGeneric(Container cont, String id, WebServicesTestUtils.checkStringMatch("nodeId", nmContext.getNodeId() .toString(), nodeId); assertEquals("totalMemoryNeededMB wrong", 0, totalMemoryNeededMB); - String shortLink = ujoin("containerlogs", cont.getContainerID().toString(), + String shortLink = ujoin("containerlogs", cont.getContainer().getId().toString(), cont.getUser()); assertTrue("containerLogsLink wrong", logsLink.contains(shortLink)); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java index 517d7fb..71bfb46 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java @@ -178,17 +178,8 @@ private void setupTokensAndEnv( Map environment = container.getEnvironment(); environment.put(ApplicationConstants.APPLICATION_WEB_PROXY_BASE_ENV, application.getWebProxyBase()); - // Set the AppAttemptId, containerId, NMHTTPAdress, AppSubmitTime to be + // Set the AppAttemptId, AppSubmitTime to be // consumable by the AM. - environment.put(ApplicationConstants.AM_CONTAINER_ID_ENV, - containerID.toString()); - environment.put(ApplicationConstants.NM_HOST_ENV, masterContainer - .getNodeId().getHost()); - environment.put(ApplicationConstants.NM_PORT_ENV, - String.valueOf(masterContainer.getNodeId().getPort())); - String parts[] = - masterContainer.getNodeHttpAddress().split(":"); - environment.put(ApplicationConstants.NM_HTTP_PORT_ENV, parts[1]); ApplicationId applicationId = application.getAppAttemptId().getApplicationId(); environment.put( diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java index 5c6247b..876d8a9 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java @@ -38,7 +38,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; -import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.log4j.Level; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; @@ -71,17 +70,17 @@ launched = true; Map env = request.getContainerLaunchContext().getEnvironment(); - containerIdAtContainerManager = - env.get(ApplicationConstants.AM_CONTAINER_ID_ENV); ContainerId containerId = - ConverterUtils.toContainerId(containerIdAtContainerManager); + request.getContainer().getId(); + containerIdAtContainerManager = containerId.toString(); attemptIdAtContainerManager = containerId.getApplicationAttemptId().toString(); - nmHostAtContainerManager = env.get(ApplicationConstants.NM_HOST_ENV); + nmHostAtContainerManager = request.getContainer().getNodeId().getHost(); nmPortAtContainerManager = - Integer.parseInt(env.get(ApplicationConstants.NM_PORT_ENV)); + request.getContainer().getNodeId().getPort(); nmHttpPortAtContainerManager = - Integer.parseInt(env.get(ApplicationConstants.NM_HTTP_PORT_ENV)); + Integer.parseInt(request.getContainer().getNodeHttpAddress() + .split(":")[1]); submitTimeAtContainerManager = Long.parseLong(env.get(ApplicationConstants.APP_SUBMIT_TIME_ENV)); maxAppAttempts =