diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java index 1f9686a..f3d3b1e 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java @@ -52,6 +52,7 @@ import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.YarnException; import org.apache.hadoop.yarn.api.ApplicationConstants; +import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; import org.apache.hadoop.yarn.service.AbstractService; /** @@ -280,7 +281,7 @@ private void runSubtask(org.apache.hadoop.mapred.Task task, // Use the AM's local dir env to generate the intermediate step // output files String[] localSysDirs = StringUtils.getTrimmedStrings( - System.getenv(ApplicationConstants.LOCAL_DIR_ENV)); + System.getenv(Environment.LOCAL_DIRS.name())); conf.setStrings(MRConfig.LOCAL_DIR, localSysDirs); LOG.info(MRConfig.LOCAL_DIR + " for uber task: " + conf.get(MRConfig.LOCAL_DIR)); diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java index 1052324..7cba10e 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java @@ -111,8 +111,6 @@ public static void setVMEnv(Map environment, MRJobConfig.STDERR_LOGFILE_ENV, getTaskLogFile(TaskLog.LogName.STDERR) ); - environment.put(MRJobConfig.APPLICATION_ATTEMPT_ID_ENV, - conf.get(MRJobConfig.APPLICATION_ATTEMPT_ID).toString()); } private static String getChildJavaOpts(JobConf jobConf, boolean isMapTask) { diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java index 2784788..6c1382d 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java @@ -57,6 +57,9 @@ import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler; import org.apache.hadoop.yarn.api.ApplicationConstants; +import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.log4j.LogManager; /** @@ -216,7 +219,7 @@ public Object run() throws Exception { */ private static void configureLocalDirs(Task task, JobConf job) throws IOException { String[] localSysDirs = StringUtils.getTrimmedStrings( - System.getenv(ApplicationConstants.LOCAL_DIR_ENV)); + System.getenv(Environment.LOCAL_DIRS.name())); job.setStrings(MRConfig.LOCAL_DIR, localSysDirs); LOG.info(MRConfig.LOCAL_DIR + " for child: " + job.get(MRConfig.LOCAL_DIR)); LocalDirAllocator lDirAlloc = new LocalDirAllocator(MRConfig.LOCAL_DIR); @@ -256,12 +259,14 @@ private static JobConf configureTask(Task task, Credentials credentials, final JobConf job = new JobConf(MRJobConfig.JOB_CONF_FILE); job.setCredentials(credentials); - String appAttemptIdEnv = System - .getenv(MRJobConfig.APPLICATION_ATTEMPT_ID_ENV); - LOG.debug("APPLICATION_ATTEMPT_ID: " + appAttemptIdEnv); + ApplicationAttemptId appAttemptId = + ConverterUtils.toContainerId( + System.getenv(Environment.CONTAINER_ID.name())) + .getApplicationAttemptId(); + LOG.debug("APPLICATION_ATTEMPT_ID: " + appAttemptId); // Set it in conf, so as to be able to be used the the OutputCommitter. - job.setInt(MRJobConfig.APPLICATION_ATTEMPT_ID, Integer - .parseInt(appAttemptIdEnv)); + job.setInt(MRJobConfig.APPLICATION_ATTEMPT_ID, + appAttemptId.getAttemptId()); // set tcp nodelay job.setBoolean("ipc.client.tcpnodelay", true); diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java index 45d6e9e..eace269 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java @@ -116,6 +116,7 @@ import org.apache.hadoop.yarn.YarnException; import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler; import org.apache.hadoop.yarn.api.ApplicationConstants; +import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -1270,22 +1271,22 @@ public static void main(String[] args) { try { Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler()); String containerIdStr = - System.getenv(ApplicationConstants.AM_CONTAINER_ID_ENV); - String nodeHostString = System.getenv(ApplicationConstants.NM_HOST_ENV); - String nodePortString = System.getenv(ApplicationConstants.NM_PORT_ENV); + System.getenv(Environment.CONTAINER_ID.name()); + String nodeHostString = System.getenv(Environment.NM_HOST.name()); + String nodePortString = System.getenv(Environment.NM_PORT.name()); String nodeHttpPortString = - System.getenv(ApplicationConstants.NM_HTTP_PORT_ENV); + System.getenv(Environment.NM_HTTP_PORT.name()); String appSubmitTimeStr = System.getenv(ApplicationConstants.APP_SUBMIT_TIME_ENV); String maxAppAttempts = System.getenv(ApplicationConstants.MAX_APP_ATTEMPTS_ENV); validateInputParam(containerIdStr, - ApplicationConstants.AM_CONTAINER_ID_ENV); - validateInputParam(nodeHostString, ApplicationConstants.NM_HOST_ENV); - validateInputParam(nodePortString, ApplicationConstants.NM_PORT_ENV); + Environment.CONTAINER_ID.name()); + validateInputParam(nodeHostString, Environment.NM_HOST.name()); + validateInputParam(nodePortString, Environment.NM_PORT.name()); validateInputParam(nodeHttpPortString, - ApplicationConstants.NM_HTTP_PORT_ENV); + Environment.NM_HTTP_PORT.name()); validateInputParam(appSubmitTimeStr, ApplicationConstants.APP_SUBMIT_TIME_ENV); validateInputParam(maxAppAttempts, diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java index ba20ee5..0c1c6fd 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java @@ -569,8 +569,6 @@ public static final String STDOUT_LOGFILE_ENV = "STDOUT_LOGFILE_ENV"; public static final String STDERR_LOGFILE_ENV = "STDERR_LOGFILE_ENV"; - public static final String APPLICATION_ATTEMPT_ID_ENV = "APPLICATION_ATTEMPT_ID_ENV"; - // This should be the directory where splits file gets localized on the node // running ApplicationMaster. public static final String JOB_SUBMIT_DIR = "jobSubmitDir"; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java index 02b15c4..8fac8cb 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java @@ -38,30 +38,6 @@ "AppClientSecretEnv"; /** - * The environment variable for CONTAINER_ID. Set in AppMaster environment - * only - */ - public static final String AM_CONTAINER_ID_ENV = "AM_CONTAINER_ID"; - - /** - * The environment variable for the NM_HOST. Set in the AppMaster environment - * only - */ - public static final String NM_HOST_ENV = "NM_HOST"; - - /** - * The environment variable for the NM_PORT. Set in the AppMaster environment - * only - */ - public static final String NM_PORT_ENV = "NM_PORT"; - - /** - * The environment variable for the NM_HTTP_PORT. Set in the AppMaster environment - * only - */ - public static final String NM_HTTP_PORT_ENV = "NM_HTTP_PORT"; - - /** * The environment variable for APP_SUBMIT_TIME. Set in AppMaster environment * only */ @@ -70,8 +46,6 @@ public static final String CONTAINER_TOKEN_FILE_ENV_NAME = UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION; - public static final String LOCAL_DIR_ENV = "YARN_LOCAL_DIRS"; - /** * The environmental variable for APPLICATION_WEB_PROXY_BASE. Set in * ApplicationMaster's environment only. This states that for all non-relative @@ -177,7 +151,37 @@ /** * $HADOOP_YARN_HOME */ - HADOOP_YARN_HOME("HADOOP_YARN_HOME"); + HADOOP_YARN_HOME("HADOOP_YARN_HOME"), + + /** + * $CONTAINER_ID + * Final, exported by NodeManager and non-modifiable by users. + */ + CONTAINER_ID("CONTAINER_ID"), + + /** + * $NM_HOST + * Final, exported by NodeManager and non-modifiable by users. + */ + NM_HOST("NM_HOST"), + + /** + * $NM_HTTP_PORT + * Final, exported by NodeManager and non-modifiable by users. + */ + NM_HTTP_PORT("NM_HTTP_PORT"), + + /** + * $NM_PORT + * Final, exported by NodeManager and non-modifiable by users. + */ + NM_PORT("NM_PORT"), + + /** + * $LOCAL_DIRS + * Final, exported by NodeManager and non-modifiable by users. + */ + LOCAL_DIRS("LOCAL_DIRS"); private final String variable; private Environment(String variable) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java index 965a8aa..d4abc50 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java @@ -45,6 +45,7 @@ import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.yarn.api.AMRMProtocol; import org.apache.hadoop.yarn.api.ApplicationConstants; +import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; import org.apache.hadoop.yarn.api.ContainerExitStatus; import org.apache.hadoop.yarn.api.ContainerManager; @@ -320,7 +321,7 @@ public boolean init(String[] args) throws ParseException, IOException { Map envs = System.getenv(); - if (!envs.containsKey(ApplicationConstants.AM_CONTAINER_ID_ENV)) { + if (!envs.containsKey(Environment.CONTAINER_ID.name())) { if (cliParser.hasOption("app_attempt_id")) { String appIdStr = cliParser.getOptionValue("app_attempt_id", ""); appAttemptID = ConverterUtils.toApplicationAttemptId(appIdStr); @@ -330,7 +331,7 @@ public boolean init(String[] args) throws ParseException, IOException { } } else { ContainerId containerId = ConverterUtils.toContainerId(envs - .get(ApplicationConstants.AM_CONTAINER_ID_ENV)); + .get(Environment.CONTAINER_ID.name())); appAttemptID = containerId.getApplicationAttemptId(); } @@ -338,16 +339,16 @@ public boolean init(String[] args) throws ParseException, IOException { throw new RuntimeException(ApplicationConstants.APP_SUBMIT_TIME_ENV + " not set in the environment"); } - if (!envs.containsKey(ApplicationConstants.NM_HOST_ENV)) { - throw new RuntimeException(ApplicationConstants.NM_HOST_ENV + if (!envs.containsKey(Environment.NM_HOST.name())) { + throw new RuntimeException(Environment.NM_HOST.name() + " not set in the environment"); } - if (!envs.containsKey(ApplicationConstants.NM_HTTP_PORT_ENV)) { - throw new RuntimeException(ApplicationConstants.NM_HTTP_PORT_ENV + if (!envs.containsKey(Environment.NM_HTTP_PORT.name())) { + throw new RuntimeException(Environment.NM_HTTP_PORT + " not set in the environment"); } - if (!envs.containsKey(ApplicationConstants.NM_PORT_ENV)) { - throw new RuntimeException(ApplicationConstants.NM_PORT_ENV + if (!envs.containsKey(Environment.NM_PORT.name())) { + throw new RuntimeException(Environment.NM_PORT.name() + " not set in the environment"); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java index 605dc3b..85388e0 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java @@ -37,6 +37,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.ApplicationConstants; +import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -58,7 +59,7 @@ * creates a new application on the RM and negotiates a new attempt id. Then it * waits for the RM app state to reach be YarnApplicationState.ACCEPTED after * which it spawns the AM in another process and passes it the container id via - * env variable ApplicationConstants.AM_CONTAINER_ID_ENV. The AM can be in any + * env variable Environment.CONTAINER_ID. The AM can be in any * language. The AM can register with the RM using the attempt id obtained * from the container id and proceed as normal. * The client redirects app stdout and stderr to its own stdout and @@ -190,10 +191,11 @@ public void launchAM(ApplicationAttemptId attemptId) throws IOException { containerId.setId(0); String hostname = InetAddress.getLocalHost().getHostName(); - envAMList.add(ApplicationConstants.AM_CONTAINER_ID_ENV + "=" + containerId); - envAMList.add(ApplicationConstants.NM_HOST_ENV + "=" + hostname); - envAMList.add(ApplicationConstants.NM_HTTP_PORT_ENV + "=0"); - envAMList.add(ApplicationConstants.NM_PORT_ENV + "=0"); + envAMList.add(Environment.CONTAINER_ID.name() + "=" + containerId); + envAMList.add(Environment.NM_HOST.name() + "=" + hostname); + envAMList.add(Environment.NM_HTTP_PORT.name() + "=0"); + envAMList.add(Environment.NM_PORT.name() + "=0"); + envAMList.add(Environment.LOCAL_DIRS.name() + "= /tmp"); envAMList.add(ApplicationConstants.APP_SUBMIT_TIME_ENV + "=" + System.currentTimeMillis()); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java index e42d74d..89032f2 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java @@ -113,13 +113,13 @@ public int launchContainer(Container container, List localDirs, List logDirs) throws IOException { FsPermission dirPerm = new FsPermission(APPDIR_PERM); - ContainerId containerId = container.getContainerID(); + ContainerId containerId = container.getContainer().getId(); // create container dirs on all disks String containerIdStr = ConverterUtils.toString(containerId); String appIdStr = ConverterUtils.toString( - container.getContainerID().getApplicationAttemptId(). + containerId.getApplicationAttemptId(). getApplicationId()); for (String sLocalDir : localDirs) { Path usersdir = new Path(sLocalDir, ContainerLocalizer.USERCACHE); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java index e0a3582..0c6b502 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java @@ -216,11 +216,11 @@ public int launchContainer(Container container, String user, String appId, Path containerWorkDir, List localDirs, List logDirs) throws IOException { - ContainerId containerId = container.getContainerID(); + ContainerId containerId = container.getContainer().getId(); String containerIdStr = ConverterUtils.toString(containerId); resourcesHandler.preExecute(containerId, - container.getResource()); + container.getContainer().getResource()); String resourcesOptions = resourcesHandler.getResourcesOption( containerId); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationContainerInitEvent.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationContainerInitEvent.java index 88be9d1..b74be63 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationContainerInitEvent.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationContainerInitEvent.java @@ -36,8 +36,8 @@ final Container container; public ApplicationContainerInitEvent(Container container) { - super(container.getContainerID().getApplicationAttemptId().getApplicationId(), - ApplicationEventType.INIT_CONTAINER); + super(container.getContainer().getId().getApplicationAttemptId() + .getApplicationId(), ApplicationEventType.INIT_CONTAINER); this.container = container; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java index e48e2fa..4e4cf0a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java @@ -274,14 +274,14 @@ public void transition(ApplicationImpl app, ApplicationEvent event) { ApplicationContainerInitEvent initEvent = (ApplicationContainerInitEvent) event; Container container = initEvent.getContainer(); - app.containers.put(container.getContainerID(), container); - LOG.info("Adding " + container.getContainerID() + app.containers.put(container.getContainer().getId(), container); + LOG.info("Adding " + container.getContainer().getId() + " to application " + app.toString()); switch (app.getApplicationState()) { case RUNNING: app.dispatcher.getEventHandler().handle(new ContainerInitEvent( - container.getContainerID())); + container.getContainer().getId())); break; case INITING: case NEW: @@ -302,7 +302,7 @@ public void transition(ApplicationImpl app, ApplicationEvent event) { // Start all the containers waiting for ApplicationInit for (Container container : app.containers.values()) { app.dispatcher.getEventHandler().handle(new ContainerInitEvent( - container.getContainerID())); + container.getContainer().getId())); } } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java index a43e1b7..c975101 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java @@ -25,12 +25,11 @@ import org.apache.hadoop.security.Credentials; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerStatus; -import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.event.EventHandler; public interface Container extends EventHandler { - org.apache.hadoop.yarn.api.records.ContainerId getContainerID(); + org.apache.hadoop.yarn.api.records.Container getContainer(); String getUser(); @@ -46,5 +45,4 @@ String toString(); - Resource getResource(); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java index 961c454..1195763 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java @@ -42,7 +42,6 @@ import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; -import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.EventHandler; @@ -313,16 +312,6 @@ public ContainerImpl(Configuration conf, } @Override - public ContainerId getContainerID() { - this.readLock.lock(); - try { - return this.container.getId(); - } finally { - this.readLock.unlock(); - } - } - - @Override public String getUser() { this.readLock.lock(); try { @@ -385,10 +374,10 @@ public ContainerStatus cloneAndGetContainerStatus() { } @Override - public Resource getResource() { + public org.apache.hadoop.yarn.api.records.Container getContainer() { this.readLock.lock(); try { - return this.container.getResource(); + return this.container; } finally { this.readLock.unlock(); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java index d48a4b7..59a4c43 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java @@ -118,7 +118,7 @@ public Integer call() { final ContainerLaunchContext launchContext = container.getLaunchContext(); final Map> localResources = container.getLocalizedResources(); - ContainerId containerID = container.getContainerID(); + ContainerId containerID = container.getContainer().getId(); String containerIdStr = ConverterUtils.toString(containerID); final String user = launchContext.getUser(); final List command = launchContext.getCommands(); @@ -299,7 +299,7 @@ public Integer call() { * @throws IOException */ public void cleanupContainer() throws IOException { - ContainerId containerId = container.getContainerID(); + ContainerId containerId = container.getContainer().getId(); String containerIdStr = ConverterUtils.toString(containerId); LOG.info("Cleaning up container " + containerIdStr); @@ -370,7 +370,7 @@ public void cleanupContainer() throws IOException { */ private String getContainerPid(Path pidFilePath) throws Exception { String containerIdStr = - ConverterUtils.toString(container.getContainerID()); + ConverterUtils.toString(container.getContainer().getId()); String processId = null; LOG.debug("Accessing pid for container " + containerIdStr + " from pid file " + pidFilePath); @@ -547,6 +547,21 @@ public void sanitizeEnv(Map environment, * Non-modifiable environment variables */ + environment.put(Environment.CONTAINER_ID.name(), container + .getContainer().getId().toString()); + + environment.put(Environment.NM_PORT.name(), + String.valueOf(container.getContainer().getNodeId().getPort())); + + environment.put(Environment.NM_HOST.name(), container.getContainer() + .getNodeId().getHost()); + + environment.put(Environment.NM_HTTP_PORT.name(), container.getContainer() + .getNodeHttpAddress().split(":")[1]); + + environment.put(Environment.LOCAL_DIRS.name(), + StringUtils.join(",", appDirs)); + putEnvIfNotNull(environment, Environment.USER.name(), container.getUser()); putEnvIfNotNull(environment, @@ -566,11 +581,6 @@ public void sanitizeEnv(Map environment, Environment.HADOOP_CONF_DIR.name(), System.getenv(Environment.HADOOP_CONF_DIR.name()) ); - - putEnvIfNotNull(environment, - ApplicationConstants.LOCAL_DIR_ENV, - StringUtils.join(",", appDirs) - ); if (!Shell.WINDOWS) { environment.put("JVM_PID", "$$"); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java index 1e3c18b..22cffb9 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java @@ -111,7 +111,7 @@ public void stop() { public void handle(ContainersLauncherEvent event) { // TODO: ContainersLauncher launches containers one by one!! Container container = event.getContainer(); - ContainerId containerId = container.getContainerID(); + ContainerId containerId = container.getContainer().getId(); switch (event.getType()) { case LAUNCH_CONTAINER: Application app = diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java index 0d1d922..f9a6778 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java @@ -359,13 +359,15 @@ private void handleInitContainerResources( ContainerLocalizationRequestEvent rsrcReqs) { Container c = rsrcReqs.getContainer(); LocalizerContext ctxt = new LocalizerContext( - c.getUser(), c.getContainerID(), c.getCredentials()); + c.getUser(), c.getContainer().getId(), c.getCredentials()); Map> rsrcs = rsrcReqs.getRequestedResources(); for (Map.Entry> e : rsrcs.entrySet()) { - LocalResourcesTracker tracker = getLocalResourcesTracker(e.getKey(), c.getUser(), - c.getContainerID().getApplicationAttemptId().getApplicationId()); + LocalResourcesTracker tracker = + getLocalResourcesTracker(e.getKey(), c.getUser(), + c.getContainer().getId().getApplicationAttemptId() + .getApplicationId()); for (LocalResourceRequest req : e.getValue()) { tracker.handle(new ResourceRequestEvent(req, e.getKey(), ctxt)); } @@ -394,19 +396,21 @@ private void handleCleanupContainerResources( for (Map.Entry> e : rsrcs.entrySet()) { LocalResourcesTracker tracker = getLocalResourcesTracker(e.getKey(), c.getUser(), - c.getContainerID().getApplicationAttemptId().getApplicationId()); + c.getContainer().getId().getApplicationAttemptId() + .getApplicationId()); for (LocalResourceRequest req : e.getValue()) { - tracker.handle(new ResourceReleaseEvent(req, c.getContainerID())); + tracker.handle(new ResourceReleaseEvent(req, + c.getContainer().getId())); } } - String locId = ConverterUtils.toString(c.getContainerID()); + String locId = ConverterUtils.toString(c.getContainer().getId()); localizerTracker.cleanupPrivLocalizers(locId); // Delete the container directories String userName = c.getUser(); String containerIDStr = c.toString(); String appIDStr = ConverterUtils.toString( - c.getContainerID().getApplicationAttemptId().getApplicationId()); + c.getContainer().getId().getApplicationAttemptId().getApplicationId()); for (String localDir : dirsHandler.getLocalDirs()) { // Delete the user-owned container-dir @@ -425,8 +429,9 @@ private void handleCleanupContainerResources( delService.delete(null, containerSysDir, new Path[] {}); } - dispatcher.getEventHandler().handle(new ContainerEvent(c.getContainerID(), - ContainerEventType.CONTAINER_RESOURCES_CLEANEDUP)); + dispatcher.getEventHandler().handle( + new ContainerEvent(c.getContainer().getId(), + ContainerEventType.CONTAINER_RESOURCES_CLEANEDUP)); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java index 712d8ff..379d998 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java @@ -60,7 +60,7 @@ public ContainerInfo(final Context nmContext, final Container container) { public ContainerInfo(final Context nmContext, final Container container, String requestUri, String pathPrefix) { - this.id = container.getContainerID().toString(); + this.id = container.getContainer().getId().toString(); this.nodeId = nmContext.getNodeId().toString(); ContainerStatus containerData = container.cloneAndGetContainerStatus(); this.exitCode = containerData.getExitStatus(); @@ -74,7 +74,7 @@ public ContainerInfo(final Context nmContext, final Container container, } this.user = container.getUser(); - Resource res = container.getResource(); + Resource res = container.getContainer().getResource(); if (res != null) { this.totalMemoryNeededMB = res.getMemory(); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java index d848405..3284634 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java @@ -88,10 +88,10 @@ public void handle(LocalizationEvent event) { .getRequestedResources().values()) { for (LocalResourceRequest req : rc) { LOG.info("DEBUG: " + req + ":" - + rsrcReqs.getContainer().getContainerID()); + + rsrcReqs.getContainer().getContainer().getId()); dispatcher.getEventHandler().handle( new ContainerResourceLocalizedEvent(rsrcReqs.getContainer() - .getContainerID(), req, new Path("file:///local" + .getContainer().getId(), req, new Path("file:///local" + req.getPath().toUri().getPath()))); } } @@ -101,7 +101,7 @@ public void handle(LocalizationEvent event) { ((ContainerLocalizationEvent) event).getContainer(); // TODO: delete the container dir this.dispatcher.getEventHandler().handle( - new ContainerEvent(container.getContainerID(), + new ContainerEvent(container.getContainer().getId(), ContainerEventType.CONTAINER_RESOURCES_CLEANEDUP)); break; case DESTROY_APPLICATION_RESOURCES: @@ -130,7 +130,7 @@ protected ContainersLauncher createContainersLauncher(Context context, @Override public void handle(ContainersLauncherEvent event) { Container container = event.getContainer(); - ContainerId containerId = container.getContainerID(); + ContainerId containerId = container.getContainer().getId(); switch (event.getType()) { case LAUNCH_CONTAINER: dispatcher.getEventHandler().handle( diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java index 02c2001..805f294 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java @@ -186,7 +186,10 @@ private int runAndBlock(ContainerId cId, String ... cmd) throws IOException { ContainerLaunchContext context = mock(ContainerLaunchContext.class); HashMap env = new HashMap(); - when(container.getContainerID()).thenReturn(cId); + org.apache.hadoop.yarn.api.records.Container containerAPI = + mock(org.apache.hadoop.yarn.api.records.Container.class); + when(container.getContainer()).thenReturn(containerAPI); + when(container.getContainer().getId()).thenReturn(cId); when(container.getLaunchContext()).thenReturn(context); when(context.getEnvironment()).thenReturn(env); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java index e6977c8..118d58a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java @@ -107,7 +107,10 @@ public void testContainerLaunch() throws IOException { ContainerLaunchContext context = mock(ContainerLaunchContext.class); HashMap env = new HashMap(); - when(container.getContainerID()).thenReturn(cId); + org.apache.hadoop.yarn.api.records.Container containerAPI = + mock(org.apache.hadoop.yarn.api.records.Container.class); + when(container.getContainer()).thenReturn(containerAPI); + when(container.getContainer().getId()).thenReturn(cId); when(container.getLaunchContext()).thenReturn(context); when(cId.toString()).thenReturn(containerId); @@ -225,7 +228,10 @@ public void testContainerLaunchError() throws IOException { ContainerLaunchContext context = mock(ContainerLaunchContext.class); HashMap env = new HashMap(); - when(container.getContainerID()).thenReturn(cId); + org.apache.hadoop.yarn.api.records.Container containerAPI = + mock(org.apache.hadoop.yarn.api.records.Container.class); + when(container.getContainer()).thenReturn(containerAPI); + when(container.getContainer().getId()).thenReturn(cId); when(container.getLaunchContext()).thenReturn(context); when(cId.toString()).thenReturn(containerId); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java index df170c7..1792988 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java @@ -54,6 +54,7 @@ import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; +import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.URL; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -178,6 +179,9 @@ private void startContainers(NodeManager nm) throws IOException { Container mockContainer = mock(Container.class); when(mockContainer.getId()).thenReturn(cId); + NodeId nodeId = BuilderUtils.newNodeId("localhost", 1234); + when(mockContainer.getNodeId()).thenReturn(nodeId); + when(mockContainer.getNodeHttpAddress()).thenReturn("localhost:12345"); containerLaunchContext.setUser(user); URL localResourceUri = diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java index b5a4d8c..981ab39 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java @@ -150,6 +150,10 @@ public void testContainerSetup() throws IOException, InterruptedException { when(mockContainer.getId()).thenReturn(cId); when(mockContainer.getResource()).thenReturn( BuilderUtils.newResource(512, 1)); + when(mockContainer.getNodeId()).thenReturn(context.getNodeId()); + when(mockContainer.getNodeId()).thenReturn(context.getNodeId()); + when(mockContainer.getNodeHttpAddress()).thenReturn( + context.getNodeId().getHost() + ":12345"); StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class); @@ -245,6 +249,10 @@ public void testContainerLaunchAndStop() throws IOException, when(mockContainer.getId()).thenReturn(cId); when(mockContainer.getResource()).thenReturn( BuilderUtils.newResource(100, 1)); // MB + when(mockContainer.getNodeId()).thenReturn(context.getNodeId()); + when(mockContainer.getNodeHttpAddress()).thenReturn( + context.getNodeId().getHost() + ":12345"); + StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class); startRequest.setContainerLaunchContext(containerLaunchContext); startRequest.setContainer(mockContainer); @@ -352,7 +360,9 @@ private void testContainerLaunchAndExit(int exitCode) throws IOException, Interr when(mockContainer.getId()).thenReturn(cId); when(mockContainer.getResource()).thenReturn( BuilderUtils.newResource(100, 1)); // MB - + when(mockContainer.getNodeId()).thenReturn(context.getNodeId()); + when(mockContainer.getNodeHttpAddress()).thenReturn( + context.getNodeId().getHost() + ":12345"); StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class); startRequest.setContainerLaunchContext(containerLaunchContext); startRequest.setContainer(mockContainer); @@ -444,6 +454,9 @@ public void testLocalFilesCleanup() throws InterruptedException, when(mockContainer.getId()).thenReturn(cId); when(mockContainer.getResource()).thenReturn( BuilderUtils.newResource(100, 1)); + when(mockContainer.getNodeId()).thenReturn(context.getNodeId()); + when(mockContainer.getNodeHttpAddress()).thenReturn( + context.getNodeId().getHost() + ":12345"); // containerLaunchContext.command = new ArrayList(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java index 3dd8d27..1050a9e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java @@ -83,7 +83,7 @@ public void testApplicationInit1() { for (int i = 0; i < wa.containers.size(); i++) { verify(wa.containerBus).handle( argThat(new ContainerInitMatcher(wa.containers.get(i) - .getContainerID()))); + .getContainer().getId()))); } } finally { if (wa != null) @@ -108,7 +108,7 @@ public void testApplicationInit2() { assertEquals(ApplicationState.RUNNING, wa.app.getApplicationState()); verify(wa.containerBus).handle( argThat(new ContainerInitMatcher(wa.containers.get(0) - .getContainerID()))); + .getContainer().getId()))); wa.initContainer(1); wa.initContainer(2); @@ -118,7 +118,7 @@ public void testApplicationInit2() { for (int i = 1; i < wa.containers.size(); i++) { verify(wa.containerBus).handle( argThat(new ContainerInitMatcher(wa.containers.get(i) - .getContainerID()))); + .getContainer().getId()))); } } finally { if (wa != null) @@ -233,7 +233,7 @@ public void testAppFinishedOnRunningContainers() { for (int i = 1; i < wa.containers.size(); i++) { verify(wa.containerBus).handle( argThat(new ContainerKillMatcher(wa.containers.get(i) - .getContainerID()))); + .getContainer().getId()))); } wa.containerFinished(1); @@ -354,7 +354,7 @@ public void testAppFinishedOnIniting() { verify(wa.containerBus).handle( argThat(new ContainerKillMatcher(wa.containers.get(0) - .getContainerID()))); + .getContainer().getId()))); assertEquals(ApplicationState.FINISHING_CONTAINERS_WAIT, wa.app.getApplicationState()); @@ -487,7 +487,7 @@ public void initContainer(int containerNum) { public void containerFinished(int containerNum) { app.handle(new ApplicationContainerFinishedEvent(containers.get( - containerNum).getContainerID())); + containerNum).getContainer().getId())); drainDispatcherEvents(); } @@ -514,7 +514,10 @@ private Container createMockedContainer(ApplicationId appId, int containerId) { BuilderUtils.newApplicationAttemptId(appId, 1); ContainerId cId = BuilderUtils.newContainerId(appAttemptId, containerId); Container c = mock(Container.class); - when(c.getContainerID()).thenReturn(cId); + org.apache.hadoop.yarn.api.records.Container containerAPI = + mock(org.apache.hadoop.yarn.api.records.Container.class); + when(c.getContainer()).thenReturn(containerAPI); + when(c.getContainer().getId()).thenReturn(cId); ContainerLaunchContext launchContext = mock(ContainerLaunchContext.class); when(c.getLaunchContext()).thenReturn(launchContext); when(launchContext.getApplicationACLs()).thenReturn( diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java index 230ce46..c98379c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java @@ -376,7 +376,7 @@ public boolean matches(Object o) { public boolean matches(Object o) { ContainersLauncherEvent evt = (ContainersLauncherEvent) o; return evt.getType() == ContainersLauncherEventType.LAUNCH_CONTAINER - && wcf.cId == evt.getContainer().getContainerID(); + && wcf.cId == evt.getContainer().getContainer().getId(); } }; verify(wc.launcherBus).handle(argThat(matchesLaunchReq)); @@ -639,7 +639,7 @@ public void initContainer() { Path p = new Path(cache, rsrc.getKey()); localPaths.put(p, Arrays.asList(rsrc.getKey())); // rsrc copied to p - c.handle(new ContainerResourceLocalizedEvent(c.getContainerID(), + c.handle(new ContainerResourceLocalizedEvent(c.getContainer().getId(), req, p)); } drainDispatcherEvents(); @@ -662,7 +662,8 @@ public void failLocalizeSpecificResource(String rsrcKey) LocalResource rsrc = localResources.get(rsrcKey); LocalResourceRequest req = new LocalResourceRequest(rsrc); Exception e = new Exception("Fake localization error"); - c.handle(new ContainerResourceFailedEvent(c.getContainerID(), req, e)); + c.handle(new ContainerResourceFailedEvent(c.getContainer() + .getId(), req, e)); drainDispatcherEvents(); } @@ -677,7 +678,7 @@ public void failLocalizeResources(int failRsrcCount) ++counter; LocalResourceRequest req = new LocalResourceRequest(rsrc.getValue()); Exception e = new Exception("Fake localization error"); - c.handle(new ContainerResourceFailedEvent(c.getContainerID(), + c.handle(new ContainerResourceFailedEvent(c.getContainer().getId(), req, e)); } drainDispatcherEvents(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java index edb9abd..0e07efb 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java @@ -37,6 +37,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.UnsupportedFileSystemException; import org.apache.hadoop.util.Shell; +import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; @@ -51,14 +52,13 @@ import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; -import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.URL; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode; -import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.Signal; import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor; import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest; import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer; import org.apache.hadoop.yarn.util.BuilderUtils; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.LinuxResourceCalculatorPlugin; @@ -150,50 +150,17 @@ public void testSpecialCharSymlinks() throws IOException { } } } - - // this is a dirty hack - but should be ok for a unittest. - @SuppressWarnings({ "rawtypes", "unchecked" }) - public static void setNewEnvironmentHack(Map newenv) throws Exception { - try { - Class cl = Class.forName("java.lang.ProcessEnvironment"); - Field field = cl.getDeclaredField("theEnvironment"); - field.setAccessible(true); - Map env = (Map)field.get(null); - env.clear(); - env.putAll(newenv); - Field ciField = cl.getDeclaredField("theCaseInsensitiveEnvironment"); - ciField.setAccessible(true); - Map cienv = (Map)ciField.get(null); - cienv.clear(); - cienv.putAll(newenv); - } catch (NoSuchFieldException e) { - Class[] classes = Collections.class.getDeclaredClasses(); - Map env = System.getenv(); - for (Class cl : classes) { - if ("java.util.Collections$UnmodifiableMap".equals(cl.getName())) { - Field field = cl.getDeclaredField("m"); - field.setAccessible(true); - Object obj = field.get(env); - Map map = (Map) obj; - map.clear(); - map.putAll(newenv); - } - } - } - } /** * See if environment variable is forwarded using sanitizeEnv. * @throws Exception */ - @Test + @Test (timeout = 5000) public void testContainerEnvVariables() throws Exception { containerManager.start(); - Map envWithDummy = new HashMap(); - envWithDummy.putAll(System.getenv()); - envWithDummy.put(Environment.MALLOC_ARENA_MAX.name(), "99"); - setNewEnvironmentHack(envWithDummy); + ContainerLaunchContext containerLaunchContext = + recordFactory.newRecordInstance(ContainerLaunchContext.class); Container mockContainer = mock(Container.class); // ////// Construct the Container-id @@ -207,34 +174,54 @@ public void testContainerEnvVariables() throws Exception { ContainerId cId = recordFactory.newRecordInstance(ContainerId.class); cId.setApplicationAttemptId(appAttemptId); - String malloc = System.getenv(Environment.MALLOC_ARENA_MAX.name()); + when(mockContainer.getId()).thenReturn(cId); + + when(mockContainer.getNodeId()).thenReturn(context.getNodeId()); + when(mockContainer.getNodeHttpAddress()).thenReturn( + context.getNodeId().getHost() + ":12345"); + + Map userSetEnv = new HashMap(); + userSetEnv.put(Environment.CONTAINER_ID.name(), "user_set_container_id"); + userSetEnv.put(Environment.NM_HOST.name(), "user_set_NM_HOST"); + userSetEnv.put(Environment.NM_PORT.name(), "user_set_NM_PORT"); + userSetEnv.put(Environment.NM_HTTP_PORT.name(), "user_set_NM_HTTP_PORT"); + userSetEnv.put(Environment.LOCAL_DIRS.name(), "user_set_LOCAL_DIR"); + containerLaunchContext.setUser(user); + containerLaunchContext.setEnvironment(userSetEnv); + File scriptFile = Shell.appendScriptExtension(tmpDir, "scriptFile"); PrintWriter fileWriter = new PrintWriter(scriptFile); File processStartFile = new File(tmpDir, "env_vars.txt").getAbsoluteFile(); if (Shell.WINDOWS) { - fileWriter.println("@echo " + Environment.MALLOC_ARENA_MAX.$() + "> " + - processStartFile); - fileWriter.println("@echo " + cId + ">> " + processStartFile); + fileWriter.println("@echo " + Environment.CONTAINER_ID.$() + "> " + + processStartFile); + fileWriter.println("@echo " + Environment.NM_HOST.$() + ">> " + + processStartFile); + fileWriter.println("@echo " + Environment.NM_PORT.$() + ">> " + + processStartFile); + fileWriter.println("@echo " + Environment.NM_HTTP_PORT.$() + ">> " + + processStartFile); + fileWriter.println("@echo " + Environment.LOCAL_DIRS.$() + ">> " + + processStartFile); fileWriter.println("@ping -n 100 127.0.0.1 >nul"); } else { fileWriter.write("\numask 0"); // So that start file is readable by the test - fileWriter.write("\necho " + Environment.MALLOC_ARENA_MAX.$() + " > " + - processStartFile); + fileWriter.write("\necho $" + Environment.CONTAINER_ID.name() + " > " + + processStartFile); + fileWriter.write("\necho $" + Environment.NM_HOST.name() + " >> " + + processStartFile); + fileWriter.write("\necho $" + Environment.NM_PORT.name() + " >> " + + processStartFile); + fileWriter.write("\necho $" + Environment.NM_HTTP_PORT.name() + " >> " + + processStartFile); + fileWriter.write("\necho $" + Environment.LOCAL_DIRS.name() + " >> " + + processStartFile); fileWriter.write("\necho $$ >> " + processStartFile); fileWriter.write("\nexec sleep 100"); } fileWriter.close(); - assert(malloc != null && !"".equals(malloc)); - - ContainerLaunchContext containerLaunchContext = - recordFactory.newRecordInstance(ContainerLaunchContext.class); - - when(mockContainer.getId()).thenReturn(cId); - - containerLaunchContext.setUser(user); - // upload the script file so that the container can run it URL resource_alpha = ConverterUtils.getYarnUrlFromPath(localFS @@ -272,9 +259,40 @@ public void testContainerEnvVariables() throws Exception { processStartFile.exists()); // Now verify the contents of the file + List localDirs = dirsHandler.getLocalDirs(); + List appDirs = new ArrayList(localDirs.size()); + for (String localDir : localDirs) { + Path usersdir = new Path(localDir, ContainerLocalizer.USERCACHE); + Path userdir = new Path(usersdir, user); + Path appsdir = new Path(userdir, ContainerLocalizer.APPCACHE); + appDirs.add(new Path(appsdir, appId.toString())); + } BufferedReader reader = new BufferedReader(new FileReader(processStartFile)); - Assert.assertEquals(malloc, reader.readLine()); + Assert.assertEquals(cId.toString(), reader.readLine()); + Assert.assertEquals(mockContainer.getNodeId().getHost(), + reader.readLine()); + Assert.assertEquals(String.valueOf(mockContainer.getNodeId().getPort()), + reader.readLine()); + Assert.assertEquals( + String.valueOf(mockContainer.getNodeHttpAddress().split(":")[1]), + reader.readLine()); + Assert.assertEquals(StringUtils.join(",", appDirs), reader.readLine()); + + Assert.assertEquals(cId.toString(), containerLaunchContext + .getEnvironment().get(Environment.CONTAINER_ID.name())); + Assert.assertEquals(mockContainer.getNodeId().getHost(), + containerLaunchContext.getEnvironment() + .get(Environment.NM_HOST.name())); + Assert.assertEquals(String.valueOf(mockContainer.getNodeId().getPort()), + containerLaunchContext.getEnvironment().get( + Environment.NM_PORT.name())); + Assert.assertEquals( + mockContainer.getNodeHttpAddress().split(":")[1], + containerLaunchContext.getEnvironment().get( + Environment.NM_HTTP_PORT.name())); + Assert.assertEquals(StringUtils.join(",", appDirs), containerLaunchContext + .getEnvironment().get(Environment.LOCAL_DIRS.name())); // Get the pid of the process String pid = reader.readLine().trim(); // No more lines @@ -354,6 +372,9 @@ public void testDelayedKill() throws Exception { ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class); when(mockContainer.getId()).thenReturn(cId); + when(mockContainer.getNodeId()).thenReturn(context.getNodeId()); + when(mockContainer.getNodeHttpAddress()).thenReturn( + context.getNodeId().getHost() + ":12345"); containerLaunchContext.setUser(user); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java index c4273bf..90c8063 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java @@ -494,7 +494,7 @@ public boolean matches(Object o) { Thread.sleep(1000); dispatcher.await(); String appStr = ConverterUtils.toString(appId); - String ctnrStr = c.getContainerID().toString(); + String ctnrStr = c.getContainer().getId().toString(); ArgumentCaptor tokenPathCaptor = ArgumentCaptor.forClass(Path.class); verify(exec).startLocalizer(tokenPathCaptor.capture(), isA(InetSocketAddress.class), eq("user0"), eq(appStr), eq(ctnrStr), @@ -570,7 +570,7 @@ public boolean matches(Object o) { public boolean matches(Object o) { ContainerEvent evt = (ContainerEvent) o; return evt.getType() == ContainerEventType.RESOURCE_LOCALIZED - && c.getContainerID() == evt.getContainerID(); + && c.getContainer().getId() == evt.getContainerID(); } }; // total 2 resource localzation calls. one for each resource. @@ -759,11 +759,11 @@ public void testParallelDownloadAttemptsForPrivateResource() throws Exception { // Container - 1 ContainerImpl container1 = createMockContainer(user, 1); - String localizerId1 = container1.getContainerID().toString(); + String localizerId1 = container1.getContainer().getId().toString(); rls.getPrivateLocalizers().put( localizerId1, rls.new LocalizerRunner(new LocalizerContext(user, container1 - .getContainerID(), null), localizerId1)); + .getContainer().getId(), null), localizerId1)); LocalizerRunner localizerRunner1 = rls.getLocalizerRunner(localizerId1); dispatcher1.getEventHandler().handle( @@ -774,11 +774,11 @@ public void testParallelDownloadAttemptsForPrivateResource() throws Exception { // Container - 2 now makes the request. ContainerImpl container2 = createMockContainer(user, 2); - String localizerId2 = container2.getContainerID().toString(); + String localizerId2 = container2.getContainer().getId().toString(); rls.getPrivateLocalizers().put( localizerId2, rls.new LocalizerRunner(new LocalizerContext(user, container2 - .getContainerID(), null), localizerId2)); + .getContainer().getId(), null), localizerId2)); LocalizerRunner localizerRunner2 = rls.getLocalizerRunner(localizerId2); dispatcher1.getEventHandler().handle( createContainerLocalizationEvent(container2, @@ -919,11 +919,11 @@ public void testLocalResourcePath() throws Exception { // Container - 1 Container container1 = createMockContainer(user, 1); - String localizerId1 = container1.getContainerID().toString(); + String localizerId1 = container1.getContainer().getId().toString(); rls.getPrivateLocalizers().put( localizerId1, rls.new LocalizerRunner(new LocalizerContext(user, container1 - .getContainerID(), null), localizerId1)); + .getContainer().getId(), null), localizerId1)); // Creating two requests for container // 1) Private resource @@ -1314,7 +1314,10 @@ private ContainerLocalizationRequestEvent createContainerLocalizationEvent( private ContainerImpl createMockContainer(String user, int containerId) { ContainerImpl container = mock(ContainerImpl.class); - when(container.getContainerID()).thenReturn( + org.apache.hadoop.yarn.api.records.Container c = + mock(org.apache.hadoop.yarn.api.records.Container.class); + when(container.getContainer()).thenReturn(c); + when(container.getContainer().getId()).thenReturn( BuilderUtils.newContainerId(1, 1, 1, containerId)); when(container.getUser()).thenReturn(user); Credentials mockCredentials = mock(Credentials.class); @@ -1354,8 +1357,11 @@ private static Container getMockContainer(ApplicationId appId, int id) { ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(appId, 1); ContainerId cId = BuilderUtils.newContainerId(appAttemptId, id); + org.apache.hadoop.yarn.api.records.Container containerAPI = + mock(org.apache.hadoop.yarn.api.records.Container.class); + when(c.getContainer()).thenReturn(containerAPI); when(c.getUser()).thenReturn("user0"); - when(c.getContainerID()).thenReturn(cId); + when(c.getContainer().getId()).thenReturn(cId); Credentials creds = new Credentials(); creds.addToken(new Text("tok" + id), getToken(id)); when(c.getCredentials()).thenReturn(creds); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java index a27b357..ad4a818 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java @@ -213,6 +213,9 @@ public void testContainerKillOnMemoryOverflow() throws IOException, cId.setApplicationAttemptId(appAttemptId); when(mockContainer.getId()).thenReturn(cId); + when(mockContainer.getNodeId()).thenReturn(context.getNodeId()); + when(mockContainer.getNodeHttpAddress()).thenReturn( + context.getNodeId().getHost() + ":12345"); containerLaunchContext.setUser(user); URL resource_alpha = diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java index dbb50bb..f233173 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java @@ -38,6 +38,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState; import org.apache.hadoop.yarn.util.BuilderUtils; +import static org.mockito.Mockito.*; public class MockContainer implements Container { @@ -48,6 +49,7 @@ private final Map> resource = new HashMap>(); private RecordFactory recordFactory; + private org.apache.hadoop.yarn.api.records.Container mockContainer; public MockContainer(ApplicationAttemptId appAttemptId, Dispatcher dispatcher, Configuration conf, String user, @@ -62,6 +64,8 @@ public MockContainer(ApplicationAttemptId appAttemptId, launchContext.setUser(user); this.state = ContainerState.NEW; + mockContainer = mock(org.apache.hadoop.yarn.api.records.Container.class); + when(mockContainer.getId()).thenReturn(id); } public void setState(ContainerState state) { @@ -69,11 +73,6 @@ public void setState(ContainerState state) { } @Override - public ContainerId getContainerID() { - return id; - } - - @Override public String getUser() { return user; } @@ -119,8 +118,7 @@ public void handle(ContainerEvent event) { } @Override - public Resource getResource() { - return null; + public org.apache.hadoop.yarn.api.records.Container getContainer() { + return this.mockContainer; } - } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java index 39764e6..bde9fee 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java @@ -185,16 +185,18 @@ public void testNodeAppsNone() throws JSONException, Exception { app.getUser(), app.getAppId(), 1); Container container2 = new MockContainer(appAttemptId, dispatcher, conf, app.getUser(), app.getAppId(), 2); - nmContext.getContainers().put(container1.getContainerID(), container1); - nmContext.getContainers().put(container2.getContainerID(), container2); + nmContext.getContainers() + .put(container1.getContainer().getId(), container1); + nmContext.getContainers() + .put(container2.getContainer().getId(), container2); - app.getContainers().put(container1.getContainerID(), container1); - app.getContainers().put(container2.getContainerID(), container2); + app.getContainers().put(container1.getContainer().getId(), container1); + app.getContainers().put(container2.getContainer().getId(), container2); HashMap hash = new HashMap(); - hash.put(container1.getContainerID().toString(), container1 - .getContainerID().toString()); - hash.put(container2.getContainerID().toString(), container2 - .getContainerID().toString()); + hash.put(container1.getContainer().getId().toString(), container1 + .getContainer().getId().toString()); + hash.put(container2.getContainer().getId().toString(), container2 + .getContainer().getId().toString()); return hash; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java index 1f678d4..0afa735 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java @@ -186,16 +186,18 @@ public void testNodeContainersNone() throws JSONException, Exception { app.getUser(), app.getAppId(), 1); Container container2 = new MockContainer(appAttemptId, dispatcher, conf, app.getUser(), app.getAppId(), 2); - nmContext.getContainers().put(container1.getContainerID(), container1); - nmContext.getContainers().put(container2.getContainerID(), container2); + nmContext.getContainers() + .put(container1.getContainer().getId(), container1); + nmContext.getContainers() + .put(container2.getContainer().getId(), container2); - app.getContainers().put(container1.getContainerID(), container1); - app.getContainers().put(container2.getContainerID(), container2); + app.getContainers().put(container1.getContainer().getId(), container1); + app.getContainers().put(container2.getContainer().getId(), container2); HashMap hash = new HashMap(); - hash.put(container1.getContainerID().toString(), container1 - .getContainerID().toString()); - hash.put(container2.getContainerID().toString(), container2 - .getContainerID().toString()); + hash.put(container1.getContainer().getId().toString(), container1 + .getContainer().getId().toString()); + hash.put(container2.getContainer().getId().toString(), container2 + .getContainer().getId().toString()); return hash; } @@ -468,7 +470,7 @@ public void verifyNodeContainerInfoGeneric(Container cont, String id, String state, String user, int exitCode, String diagnostics, String nodeId, int totalMemoryNeededMB, String logsLink) throws JSONException, Exception { - WebServicesTestUtils.checkStringMatch("id", cont.getContainerID() + WebServicesTestUtils.checkStringMatch("id", cont.getContainer().getId() .toString(), id); WebServicesTestUtils.checkStringMatch("state", cont.getContainerState() .toString(), state); @@ -481,8 +483,9 @@ public void verifyNodeContainerInfoGeneric(Container cont, String id, WebServicesTestUtils.checkStringMatch("nodeId", nmContext.getNodeId() .toString(), nodeId); assertEquals("totalMemoryNeededMB wrong", 0, totalMemoryNeededMB); - String shortLink = ujoin("containerlogs", cont.getContainerID().toString(), - cont.getUser()); + String shortLink = + ujoin("containerlogs", cont.getContainer().getId().toString(), + cont.getUser()); assertTrue("containerLogsLink wrong", logsLink.contains(shortLink)); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java index 517d7fb..fb95550 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java @@ -178,17 +178,7 @@ private void setupTokensAndEnv( Map environment = container.getEnvironment(); environment.put(ApplicationConstants.APPLICATION_WEB_PROXY_BASE_ENV, application.getWebProxyBase()); - // Set the AppAttemptId, containerId, NMHTTPAdress, AppSubmitTime to be - // consumable by the AM. - environment.put(ApplicationConstants.AM_CONTAINER_ID_ENV, - containerID.toString()); - environment.put(ApplicationConstants.NM_HOST_ENV, masterContainer - .getNodeId().getHost()); - environment.put(ApplicationConstants.NM_PORT_ENV, - String.valueOf(masterContainer.getNodeId().getPort())); - String parts[] = - masterContainer.getNodeHttpAddress().split(":"); - environment.put(ApplicationConstants.NM_HTTP_PORT_ENV, parts[1]); + // Set AppSubmitTime and MaxAppAttempts to be consumable by the AM. ApplicationId applicationId = application.getAppAttemptId().getApplicationId(); environment.put( diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java index 5c6247b..876d8a9 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java @@ -38,7 +38,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; -import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.log4j.Level; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; @@ -71,17 +70,17 @@ launched = true; Map env = request.getContainerLaunchContext().getEnvironment(); - containerIdAtContainerManager = - env.get(ApplicationConstants.AM_CONTAINER_ID_ENV); ContainerId containerId = - ConverterUtils.toContainerId(containerIdAtContainerManager); + request.getContainer().getId(); + containerIdAtContainerManager = containerId.toString(); attemptIdAtContainerManager = containerId.getApplicationAttemptId().toString(); - nmHostAtContainerManager = env.get(ApplicationConstants.NM_HOST_ENV); + nmHostAtContainerManager = request.getContainer().getNodeId().getHost(); nmPortAtContainerManager = - Integer.parseInt(env.get(ApplicationConstants.NM_PORT_ENV)); + request.getContainer().getNodeId().getPort(); nmHttpPortAtContainerManager = - Integer.parseInt(env.get(ApplicationConstants.NM_HTTP_PORT_ENV)); + Integer.parseInt(request.getContainer().getNodeHttpAddress() + .split(":")[1]); submitTimeAtContainerManager = Long.parseLong(env.get(ApplicationConstants.APP_SUBMIT_TIME_ENV)); maxAppAttempts =