diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 46e3323..250b042 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -534,6 +534,18 @@ public static boolean isAclEnabled(Configuration conf) { public static final int DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE = 10; + /** + * The {@code AMLauncher.createAMContainerLaunchContext()} method will log the + * command being executed to the RM log if this property is true. Commands + * may contain sensitive information, such as application or service + * passwords, making logging the commands a security risk. In those cases, + * this property should be set to false. Commands are only logged at the + * debug level. + */ + public static final String RM_AMLAUNCHER_LOG_COMMAND = + RM_PREFIX + "amlauncher.log.command"; + public static final boolean DEFAULT_RM_AMLAUNCHER_LOG_COMMAND = true; + //RM delegation token related keys public static final String RM_DELEGATION_KEY_UPDATE_INTERVAL_KEY = RM_PREFIX + "delegation.key.update-interval"; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index e956507..97c297f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -299,6 +299,18 @@ + + The resource manager will log all commands being executed to the RM log + if this property is true. Commands may contain sensitive information, + such as application or service passwords, making logging the commands a + security risk. In those cases, this property should be set to false. + Commands are only logged at the debug level. + + yarn.resourcemanager.amlauncher.log.command + true + + + The class to use as the resource scheduler. yarn.resourcemanager.scheduler.class org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java index 4aace2c..644c298 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java @@ -65,7 +65,12 @@ import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.timeline.TimelineUtils; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.DEFAULT_QUEUE_NAME; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.DEFAULT_RM_AMLAUNCHER_LOG_COMMAND; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.RM_AMLAUNCHER_LOG_COMMAND; + import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Joiner; /** * The launch of the AM itself. @@ -81,6 +86,7 @@ private final AMLauncherEventType eventType; private final RMContext rmContext; private final Container masterContainer; + private final boolean logCommandLine; @SuppressWarnings("rawtypes") private final EventHandler handler; @@ -93,6 +99,8 @@ public AMLauncher(RMContext rmContext, RMAppAttempt application, this.rmContext = rmContext; this.handler = rmContext.getDispatcher().getEventHandler(); this.masterContainer = application.getMasterContainer(); + this.logCommandLine = conf.getBoolean(RM_AMLAUNCHER_LOG_COMMAND, + DEFAULT_RM_AMLAUNCHER_LOG_COMMAND); } private void connect() throws IOException { @@ -114,7 +122,7 @@ private void launch() throws IOException, YarnException { StartContainerRequest scRequest = StartContainerRequest.newInstance(launchContext, masterContainer.getContainerToken()); - List list = new ArrayList(); + List list = new ArrayList<>(); list.add(scRequest); StartContainersRequest allRequests = StartContainersRequest.newInstance(list); @@ -135,7 +143,7 @@ private void launch() throws IOException, YarnException { private void cleanup() throws IOException, YarnException { connect(); ContainerId containerId = masterContainer.getId(); - List containerIds = new ArrayList(); + List containerIds = new ArrayList<>(); containerIds.add(containerId); StopContainersRequest stopRequest = StopContainersRequest.newInstance(containerIds); @@ -188,11 +196,20 @@ private ContainerLaunchContext createAMContainerLaunchContext( // Construct the actual Container ContainerLaunchContext container = applicationMasterContext.getAMContainerSpec(); - LOG.info("Command to launch container " - + containerID - + " : " - + StringUtils.arrayToString(container.getCommands().toArray( - new String[0]))); + + if (LOG.isDebugEnabled()) { + StringBuilder message = new StringBuilder("Command to launch container "); + + message.append(containerID).append(" : "); + + if (logCommandLine) { + message.append(Joiner.on(",").join(container.getCommands())); + } else { + message.append(""); + } + + LOG.debug(message.toString()); + } // Populate the current queue name in the environment variable. setupQueueNameEnv(container, applicationMasterContext); @@ -208,9 +225,11 @@ private ContainerLaunchContext createAMContainerLaunchContext( private void setupQueueNameEnv(ContainerLaunchContext container, ApplicationSubmissionContext applicationMasterContext) { String queueName = applicationMasterContext.getQueue(); + if (queueName == null) { - queueName = YarnConfiguration.DEFAULT_QUEUE_NAME; + queueName = DEFAULT_QUEUE_NAME; } + container.getEnvironment().put(ApplicationConstants.Environment .YARN_RESOURCEMANAGER_APPLICATION_QUEUE.key(), queueName); } @@ -318,7 +337,7 @@ public void run() { switch (eventType) { case LAUNCH: try { - LOG.info("Launching master" + application.getAppAttemptId()); + LOG.info("Launching master " + application.getAppAttemptId()); launch(); handler.handle(new RMAppAttemptEvent(application.getAppAttemptId(), RMAppAttemptEventType.LAUNCHED));