diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index c927a66..ea70643 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -580,7 +580,8 @@ public SubmitApplicationResponse submitApplication( LOG.warn("Unable to get the current user.", ie); RMAuditLogger.logFailure(user, AuditConstants.SUBMIT_APP_REQUEST, ie.getMessage(), "ClientRMService", - "Exception in submitting application", applicationId, callerContext); + "Exception in submitting application", applicationId, callerContext, + submissionContext.getQueue()); throw RPCUtil.getRemoteException(ie); } @@ -623,13 +624,15 @@ public SubmitApplicationResponse submitApplication( LOG.info("Application with id " + applicationId.getId() + " submitted by user " + user); RMAuditLogger.logSuccess(user, AuditConstants.SUBMIT_APP_REQUEST, - "ClientRMService", applicationId, callerContext); + "ClientRMService", applicationId, callerContext, + submissionContext.getQueue()); } catch (YarnException e) { LOG.info("Exception in submitting application with id " + applicationId.getId(), e); RMAuditLogger.logFailure(user, AuditConstants.SUBMIT_APP_REQUEST, e.getMessage(), "ClientRMService", - "Exception in submitting application", applicationId, callerContext); + "Exception in submitting application", applicationId, callerContext, + submissionContext.getQueue()); throw e; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java index 07634c7..ee1a9fd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java @@ -39,7 +39,7 @@ static enum Keys {USER, OPERATION, TARGET, RESULT, IP, PERMISSIONS, DESCRIPTION, APPID, APPATTEMPTID, CONTAINERID, CALLERCONTEXT, CALLERSIGNATURE, QUEUENAME, - INCLUDEAPPS, INCLUDECHILDQUEUES, RECURSIVE} + INCLUDEAPPS, INCLUDECHILDQUEUES, RECURSIVE, NODELABEL} public static class AuditConstants { static final String SUCCESS = "SUCCESS"; @@ -89,7 +89,7 @@ static String createSuccessLog(String user, String operation, String target, ApplicationId appId, ApplicationAttemptId attemptId, ContainerId containerId) { return createSuccessLog(user, operation, target, appId, attemptId, - containerId, null, Server.getRemoteIp()); + containerId, null, Server.getRemoteIp(), null, null); } /** @@ -115,7 +115,7 @@ private static StringBuilder createStringBuilderForSuccessEvent(String user, static String createSuccessLog(String user, String operation, String target, ApplicationId appId, ApplicationAttemptId attemptId, ContainerId containerId, CallerContext callerContext, - InetAddress ip) { + InetAddress ip, String queueName, String partition) { StringBuilder b = createStringBuilderForSuccessEvent(user, operation, target, ip); if (appId != null) { @@ -128,6 +128,12 @@ static String createSuccessLog(String user, String operation, String target, add(Keys.CONTAINERID, containerId.toString(), b); } appendCallerContext(b, callerContext); + if (queueName != null) { + add(Keys.QUEUENAME, queueName, b); + } + if (partition != null) { + add(Keys.NODELABEL, partition, b); + } return b.toString(); } @@ -190,6 +196,30 @@ public static void logSuccess(String user, String operation, String target, } /** + * Create a readable and parseable audit log string for a successful event. + * + * @param user User who made the service request to the ResourceManager + * @param operation Operation requested by the user. + * @param target The target on which the operation is being performed. + * @param appId Application Id in which operation was performed. + * @param containerId Container Id in which operation was performed. + * @param queueName Name of queue. + * @param partition Name of labeled partition. + * + *

+ * Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter + * and hence the value fields should not contains tabs ('\t'). + */ + public static void logSuccess(String user, String operation, String target, + ApplicationId appId, ContainerId containerId, String queueName, + String partition) { + if (LOG.isInfoEnabled()) { + LOG.info(createSuccessLog(user, operation, target, appId, null, + containerId, null, null, queueName, partition)); + } + } + + /** * Create a general readable and parseable audit log string for a successful * event. * @@ -255,7 +285,15 @@ public static void logSuccess(String user, String operation, String target, ApplicationId appId, CallerContext callerContext) { if (LOG.isInfoEnabled()) { LOG.info(createSuccessLog(user, operation, target, appId, null, null, - callerContext, Server.getRemoteIp())); + callerContext, Server.getRemoteIp(), null, null)); + } + } + + public static void logSuccess(String user, String operation, String target, + ApplicationId appId, CallerContext callerContext, String queueName) { + if (LOG.isInfoEnabled()) { + LOG.info(createSuccessLog(user, operation, target, appId, null, null, + callerContext, Server.getRemoteIp(), queueName, null)); } } @@ -283,7 +321,7 @@ public static void logSuccess(String user, String operation, String target, ApplicationId appId, InetAddress ip) { if (LOG.isInfoEnabled()) { LOG.info(createSuccessLog(user, operation, target, appId, null, null, - null, ip)); + null, ip, null, null)); } } @@ -342,7 +380,7 @@ private static StringBuilder createStringBuilderForFailureLog(String user, static String createFailureLog(String user, String operation, String perm, String target, String description, ApplicationId appId, ApplicationAttemptId attemptId, ContainerId containerId, - CallerContext callerContext) { + CallerContext callerContext, String queueName) { StringBuilder b = createStringBuilderForFailureLog(user, operation, target, description, perm); if (appId != null) { @@ -355,6 +393,9 @@ static String createFailureLog(String user, String operation, String perm, add(Keys.CONTAINERID, containerId.toString(), b); } appendCallerContext(b, callerContext); + if (queueName != null) { + add(Keys.QUEUENAME, queueName, b); + } return b.toString(); } @@ -365,7 +406,7 @@ static String createFailureLog(String user, String operation, String perm, String target, String description, ApplicationId appId, ApplicationAttemptId attemptId, ContainerId containerId) { return createFailureLog(user, operation, perm, target, description, appId, - attemptId, containerId, null); + attemptId, containerId, null, null); } /** @@ -436,7 +477,16 @@ public static void logFailure(String user, String operation, String perm, CallerContext callerContext) { if (LOG.isWarnEnabled()) { LOG.warn(createFailureLog(user, operation, perm, target, description, - appId, null, null, callerContext)); + appId, null, null, callerContext, null)); + } + } + + public static void logFailure(String user, String operation, String perm, + String target, String description, ApplicationId appId, + CallerContext callerContext, String queueName) { + if (LOG.isWarnEnabled()) { + LOG.warn(createFailureLog(user, operation, perm, target, description, + appId, null, null, callerContext, queueName)); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java index 782ad38..9663719 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java @@ -166,9 +166,15 @@ public synchronized boolean containerCompleted(RMContainer rmContainer, containersToPreempt.remove(containerId); + // In order to save space in the audit log, only include the partition + // if it is not the default partition. + String containerPartition = null; + if (partition != null && !partition.isEmpty()) { + containerPartition = partition; + } RMAuditLogger.logSuccess(getUser(), AuditConstants.RELEASE_CONTAINER, "SchedulerApp", - getApplicationId(), containerId); + getApplicationId(), containerId, getQueueName(), containerPartition); // Update usage metrics Resource containerResource = rmContainer.getContainer().getResource(); @@ -230,9 +236,15 @@ public synchronized RMContainer allocate(NodeType type, FiCaSchedulerNode node, + " container=" + containerId + " host=" + container.getNodeId().getHost() + " type=" + type); } + // In order to save space in the audit log, only include the partition + // if it is not the default partition. + String partition = node.getPartition(); + if (partition == null || partition.isEmpty()) { + partition = null; + } RMAuditLogger.logSuccess(getUser(), AuditConstants.ALLOC_CONTAINER, "SchedulerApp", - getApplicationId(), containerId); + getApplicationId(), containerId, getQueueName(), partition); return rmContainer; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java index fa57dbe..fb93285 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java @@ -150,7 +150,7 @@ synchronized public void containerCompleted(RMContainer rmContainer, RMAuditLogger.logSuccess(getUser(), AuditConstants.RELEASE_CONTAINER, "SchedulerApp", - getApplicationId(), containerId); + getApplicationId(), containerId, rmContainer.getQueueName(), null); // Update usage metrics Resource containerResource = rmContainer.getContainer().getResource(); @@ -404,7 +404,7 @@ else if (allowed.equals(NodeType.RACK_LOCAL) && } RMAuditLogger.logSuccess(getUser(), AuditConstants.ALLOC_CONTAINER, "SchedulerApp", - getApplicationId(), container.getId()); + getApplicationId(), container.getId(), getQueueName(), null); return rmContainer; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java index 7ce7a49..c1d2967 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java @@ -65,6 +65,7 @@ private static final ContainerId CONTAINERID = mock(ContainerId.class); private static final String CALLER_CONTEXT = "context"; private static final byte[] CALLER_SIGNATURE = "signature".getBytes(); + private static final String PARTITION = "label1"; @Before public void setUp() throws Exception { @@ -128,6 +129,14 @@ private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId, ApplicationAttemptId attemptId, ContainerId containerId, CallerContext callerContext, InetAddress remoteIp, RMAuditLogger.ArgsBuilder args) { + testSuccessLogFormatHelper(checkIP, appId, attemptId, containerId, + callerContext, remoteIp, args, null, null); + } + + private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId, + ApplicationAttemptId attemptId, ContainerId containerId, + CallerContext callerContext, InetAddress remoteIp, + RMAuditLogger.ArgsBuilder args, String queueName, String partition) { String sLog; InetAddress tmpIp = checkIP ? remoteIp : null; if (args != null) { @@ -135,7 +144,7 @@ private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId, tmpIp, args); } else { sLog = RMAuditLogger.createSuccessLog(USER, OPERATION, TARGET, appId, - attemptId, containerId, callerContext, tmpIp); + attemptId, containerId, callerContext, tmpIp, queueName, partition); } StringBuilder expLog = new StringBuilder(); expLog.append("USER=test\t"); @@ -170,6 +179,13 @@ private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId, if (args != null) { expLog.append("\tQUEUENAME=root"); expLog.append("\tRECURSIVE=true"); + } else { + if (queueName != null) { + expLog.append("\tQUEUENAME=" + QUEUE); + } + } + if (partition != null) { + expLog.append("\tNODELABEL=" + PARTITION); } assertEquals(expLog.toString(), sLog); } @@ -251,6 +267,8 @@ private void testSuccessLogFormat(boolean checkIP) { .append(Keys.QUEUENAME, QUEUE).append(Keys.RECURSIVE, "true"); testSuccessLogFormatHelper(checkIP, null, null, null, null, Server.getRemoteIp(), args); + testSuccessLogFormatHelper(checkIP, null, null, null, null, + Server.getRemoteIp(), null, QUEUE, PARTITION); testSuccessLogFormatHelperWithIP(checkIP, APPID, ATTEMPTID, CONTAINERID); testSuccessLogNulls(checkIP); } @@ -275,7 +293,7 @@ private void testFailureLogFormatHelper(boolean checkIP, ApplicationId appId, CallerContext callerContext, RMAuditLogger.ArgsBuilder args) { String fLog = args == null ? RMAuditLogger.createFailureLog(USER, OPERATION, PERM, TARGET, DESC, - appId, attemptId, containerId, callerContext) : + appId, attemptId, containerId, callerContext, null) : RMAuditLogger.createFailureLog(USER, OPERATION, PERM, TARGET, DESC, args); StringBuilder expLog = new StringBuilder();