diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java index 7906e6f..7259226 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java @@ -347,162 +347,168 @@ public ApplicationMaster() { * @throws IOException */ public boolean init(String[] args) throws ParseException, IOException { - Options opts = new Options(); - opts.addOption("app_attempt_id", true, - "App Attempt ID. Not to be used unless for testing purposes"); - opts.addOption("shell_env", true, - "Environment for shell script. Specified as env_key=env_val pairs"); - opts.addOption("container_memory", true, - "Amount of memory in MB to be requested to run the shell command"); - opts.addOption("container_vcores", true, - "Amount of virtual cores to be requested to run the shell command"); - opts.addOption("num_containers", true, - "No. of containers on which the shell command needs to be executed"); - opts.addOption("priority", true, "Application Priority. Default 0"); - opts.addOption("debug", false, "Dump out debug information"); - - opts.addOption("help", false, "Print usage"); - CommandLine cliParser = new GnuParser().parse(opts, args); - - if (args.length == 0) { - printUsage(opts); - throw new IllegalArgumentException( - "No args specified for application master to initialize"); - } + if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) { + Options opts = new Options(); + opts.addOption("app_attempt_id", true, + "App Attempt ID. Not to be used unless for testing purposes"); + opts.addOption("shell_env", true, + "Environment for shell script. Specified as env_key=env_val pairs"); + opts.addOption("container_memory", true, + "Amount of memory in MB to be requested to run the shell command"); + opts.addOption("container_vcores", true, + "Amount of virtual cores to be requested to run the shell command"); + opts.addOption("num_containers", true, + "No. of containers on which the shell command needs to be executed"); + opts.addOption("priority", true, "Application Priority. Default 0"); + opts.addOption("debug", false, "Dump out debug information"); + + opts.addOption("help", false, "Print usage"); + CommandLine cliParser = new GnuParser().parse(opts, args); + + if (args.length == 0) { + printUsage(opts); + throw new IllegalArgumentException( + "No args specified for application master to initialize"); + } - //Check whether customer log4j.properties file exists - if (fileExist(log4jPath)) { - try { - Log4jPropertyHelper.updateLog4jConfiguration(ApplicationMaster.class, - log4jPath); - } catch (Exception e) { - LOG.warn("Can not set up custom log4j properties. " + e); + //Check whether customer log4j.properties file exists + if (fileExist(log4jPath)) { + try { + Log4jPropertyHelper.updateLog4jConfiguration(ApplicationMaster.class, + log4jPath); + } catch (Exception e) { + LOG.warn("Can not set up custom log4j properties. " + e); + } } - } - if (cliParser.hasOption("help")) { - printUsage(opts); - return false; - } + if (cliParser.hasOption("help")) { + printUsage(opts); + return false; + } - if (cliParser.hasOption("debug")) { - dumpOutDebugInfo(); - } + if (cliParser.hasOption("debug")) { + dumpOutDebugInfo(); + } - Map envs = System.getenv(); + Map envs = System.getenv(); - if (!envs.containsKey(Environment.CONTAINER_ID.name())) { - if (cliParser.hasOption("app_attempt_id")) { - String appIdStr = cliParser.getOptionValue("app_attempt_id", ""); - appAttemptID = ConverterUtils.toApplicationAttemptId(appIdStr); + if (!envs.containsKey(Environment.CONTAINER_ID.name())) { + if (cliParser.hasOption("app_attempt_id")) { + String appIdStr = cliParser.getOptionValue("app_attempt_id", ""); + appAttemptID = ConverterUtils.toApplicationAttemptId(appIdStr); + } else { + throw new IllegalArgumentException( + "Application Attempt Id not set in the environment"); + } } else { - throw new IllegalArgumentException( - "Application Attempt Id not set in the environment"); + ContainerId containerId = ConverterUtils.toContainerId(envs + .get(Environment.CONTAINER_ID.name())); + appAttemptID = containerId.getApplicationAttemptId(); } - } else { - ContainerId containerId = ConverterUtils.toContainerId(envs - .get(Environment.CONTAINER_ID.name())); - appAttemptID = containerId.getApplicationAttemptId(); - } - if (!envs.containsKey(ApplicationConstants.APP_SUBMIT_TIME_ENV)) { - throw new RuntimeException(ApplicationConstants.APP_SUBMIT_TIME_ENV - + " not set in the environment"); - } - if (!envs.containsKey(Environment.NM_HOST.name())) { - throw new RuntimeException(Environment.NM_HOST.name() - + " not set in the environment"); - } - if (!envs.containsKey(Environment.NM_HTTP_PORT.name())) { - throw new RuntimeException(Environment.NM_HTTP_PORT - + " not set in the environment"); - } - if (!envs.containsKey(Environment.NM_PORT.name())) { - throw new RuntimeException(Environment.NM_PORT.name() - + " not set in the environment"); - } + if (!envs.containsKey(ApplicationConstants.APP_SUBMIT_TIME_ENV)) { + throw new RuntimeException(ApplicationConstants.APP_SUBMIT_TIME_ENV + + " not set in the environment"); + } + if (!envs.containsKey(Environment.NM_HOST.name())) { + throw new RuntimeException(Environment.NM_HOST.name() + + " not set in the environment"); + } + if (!envs.containsKey(Environment.NM_HTTP_PORT.name())) { + throw new RuntimeException(Environment.NM_HTTP_PORT + + " not set in the environment"); + } + if (!envs.containsKey(Environment.NM_PORT.name())) { + throw new RuntimeException(Environment.NM_PORT.name() + + " not set in the environment"); + } - LOG.info("Application master for app" + ", appId=" - + appAttemptID.getApplicationId().getId() + ", clustertimestamp=" - + appAttemptID.getApplicationId().getClusterTimestamp() - + ", attemptId=" + appAttemptID.getAttemptId()); + LOG.info("Application master for app" + ", appId=" + + appAttemptID.getApplicationId().getId() + ", clustertimestamp=" + + appAttemptID.getApplicationId().getClusterTimestamp() + + ", attemptId=" + appAttemptID.getAttemptId()); - if (!fileExist(shellCommandPath) - && envs.get(DSConstants.DISTRIBUTEDSHELLSCRIPTLOCATION).isEmpty()) { - throw new IllegalArgumentException( - "No shell command or shell script specified to be executed by application master"); - } + if (!fileExist(shellCommandPath) + && envs.get(DSConstants.DISTRIBUTEDSHELLSCRIPTLOCATION).isEmpty()) { + throw new IllegalArgumentException( + "No shell command or shell script specified to be executed by application master"); + } - if (fileExist(shellCommandPath)) { - shellCommand = readContent(shellCommandPath); - } + if (fileExist(shellCommandPath)) { + shellCommand = readContent(shellCommandPath); + } - if (fileExist(shellArgsPath)) { - shellArgs = readContent(shellArgsPath); - } + if (fileExist(shellArgsPath)) { + shellArgs = readContent(shellArgsPath); + } - if (cliParser.hasOption("shell_env")) { - String shellEnvs[] = cliParser.getOptionValues("shell_env"); - for (String env : shellEnvs) { - env = env.trim(); - int index = env.indexOf('='); - if (index == -1) { - shellEnv.put(env, ""); - continue; - } - String key = env.substring(0, index); - String val = ""; - if (index < (env.length() - 1)) { - val = env.substring(index + 1); + if (cliParser.hasOption("shell_env")) { + String shellEnvs[] = cliParser.getOptionValues("shell_env"); + for (String env : shellEnvs) { + env = env.trim(); + int index = env.indexOf('='); + if (index == -1) { + shellEnv.put(env, ""); + continue; + } + String key = env.substring(0, index); + String val = ""; + if (index < (env.length() - 1)) { + val = env.substring(index + 1); + } + shellEnv.put(key, val); } - shellEnv.put(key, val); } - } - if (envs.containsKey(DSConstants.DISTRIBUTEDSHELLSCRIPTLOCATION)) { - scriptPath = envs.get(DSConstants.DISTRIBUTEDSHELLSCRIPTLOCATION); + if (envs.containsKey(DSConstants.DISTRIBUTEDSHELLSCRIPTLOCATION)) { + scriptPath = envs.get(DSConstants.DISTRIBUTEDSHELLSCRIPTLOCATION); - if (envs.containsKey(DSConstants.DISTRIBUTEDSHELLSCRIPTTIMESTAMP)) { - shellScriptPathTimestamp = Long.valueOf(envs - .get(DSConstants.DISTRIBUTEDSHELLSCRIPTTIMESTAMP)); + if (envs.containsKey(DSConstants.DISTRIBUTEDSHELLSCRIPTTIMESTAMP)) { + shellScriptPathTimestamp = Long.valueOf(envs + .get(DSConstants.DISTRIBUTEDSHELLSCRIPTTIMESTAMP)); + } + if (envs.containsKey(DSConstants.DISTRIBUTEDSHELLSCRIPTLEN)) { + shellScriptPathLen = Long.valueOf(envs + .get(DSConstants.DISTRIBUTEDSHELLSCRIPTLEN)); + } + if (!scriptPath.isEmpty() + && (shellScriptPathTimestamp <= 0 || shellScriptPathLen <= 0)) { + LOG.error("Illegal values in env for shell script path" + ", path=" + + scriptPath + ", len=" + shellScriptPathLen + ", timestamp=" + + shellScriptPathTimestamp); + throw new IllegalArgumentException( + "Illegal values in env for shell script path"); + } } - if (envs.containsKey(DSConstants.DISTRIBUTEDSHELLSCRIPTLEN)) { - shellScriptPathLen = Long.valueOf(envs - .get(DSConstants.DISTRIBUTEDSHELLSCRIPTLEN)); + + if (envs.containsKey(DSConstants.DISTRIBUTEDSHELLTIMELINEDOMAIN)) { + domainId = envs.get(DSConstants.DISTRIBUTEDSHELLTIMELINEDOMAIN); } - if (!scriptPath.isEmpty() - && (shellScriptPathTimestamp <= 0 || shellScriptPathLen <= 0)) { - LOG.error("Illegal values in env for shell script path" + ", path=" - + scriptPath + ", len=" + shellScriptPathLen + ", timestamp=" - + shellScriptPathTimestamp); + + containerMemory = Integer.parseInt(cliParser.getOptionValue( + "container_memory", "10")); + containerVirtualCores = Integer.parseInt(cliParser.getOptionValue( + "container_vcores", "1")); + numTotalContainers = Integer.parseInt(cliParser.getOptionValue( + "num_containers", "1")); + if (numTotalContainers == 0) { throw new IllegalArgumentException( - "Illegal values in env for shell script path"); + "Cannot run distributed shell with no containers"); } + requestPriority = Integer.parseInt(cliParser + .getOptionValue("priority", "0")); + + // Creating the Timeline Client + timelineClient = TimelineClient.createTimelineClient(); + timelineClient.init(conf); + timelineClient.start(); + return true; + } else { + LOG.warn("Cannot initialize App Master because the timeline service " + + "is not enabled"); + return false; } - - if (envs.containsKey(DSConstants.DISTRIBUTEDSHELLTIMELINEDOMAIN)) { - domainId = envs.get(DSConstants.DISTRIBUTEDSHELLTIMELINEDOMAIN); - } - - containerMemory = Integer.parseInt(cliParser.getOptionValue( - "container_memory", "10")); - containerVirtualCores = Integer.parseInt(cliParser.getOptionValue( - "container_vcores", "1")); - numTotalContainers = Integer.parseInt(cliParser.getOptionValue( - "num_containers", "1")); - if (numTotalContainers == 0) { - throw new IllegalArgumentException( - "Cannot run distributed shell with no containers"); - } - requestPriority = Integer.parseInt(cliParser - .getOptionValue("priority", "0")); - - // Creating the Timeline Client - timelineClient = TimelineClient.createTimelineClient(); - timelineClient.init(conf); - timelineClient.start(); - - return true; } /** diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java index 2028cc9..e8f4828 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java @@ -105,7 +105,6 @@ private DelegationTokenAuthenticator authenticator; private DelegationTokenAuthenticatedURL.Token token; private URI resURI; - private boolean isEnabled; @Private @VisibleForTesting @@ -247,12 +246,6 @@ public TimelineClientImpl() { } protected void serviceInit(Configuration conf) throws Exception { - isEnabled = conf.getBoolean( - YarnConfiguration.TIMELINE_SERVICE_ENABLED, - YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED); - if (!isEnabled) { - LOG.info("Timeline service is not enabled"); - } else { ClientConfig cc = new DefaultClientConfig(); cc.getClasses().add(YarnJacksonJaxbJsonProvider.class); connConfigurator = newConnConfigurator(conf); @@ -283,19 +276,12 @@ protected void serviceInit(Configuration conf) throws Exception { RESOURCE_URI_STR)); } LOG.info("Timeline service address: " + resURI); - } super.serviceInit(conf); } @Override public TimelinePutResponse putEntities( TimelineEntity... entities) throws IOException, YarnException { - if (!isEnabled) { - if (LOG.isDebugEnabled()) { - LOG.debug("Nothing will be put because timeline service is not enabled"); - } - return new TimelinePutResponse(); - } TimelineEntities entitiesContainer = new TimelineEntities(); entitiesContainer.addEntities(Arrays.asList(entities)); ClientResponse resp = doPosting(entitiesContainer, null); @@ -306,12 +292,6 @@ public TimelinePutResponse putEntities( @Override public void putDomain(TimelineDomain domain) throws IOException, YarnException { - if (!isEnabled) { - if (LOG.isDebugEnabled()) { - LOG.debug("Nothing will be put because timeline service is not enabled"); - } - return; - } doPosting(domain, "domain"); }