diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
index c927a66..ea70643 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
@@ -580,7 +580,8 @@ public SubmitApplicationResponse submitApplication(
LOG.warn("Unable to get the current user.", ie);
RMAuditLogger.logFailure(user, AuditConstants.SUBMIT_APP_REQUEST,
ie.getMessage(), "ClientRMService",
- "Exception in submitting application", applicationId, callerContext);
+ "Exception in submitting application", applicationId, callerContext,
+ submissionContext.getQueue());
throw RPCUtil.getRemoteException(ie);
}
@@ -623,13 +624,15 @@ public SubmitApplicationResponse submitApplication(
LOG.info("Application with id " + applicationId.getId() +
" submitted by user " + user);
RMAuditLogger.logSuccess(user, AuditConstants.SUBMIT_APP_REQUEST,
- "ClientRMService", applicationId, callerContext);
+ "ClientRMService", applicationId, callerContext,
+ submissionContext.getQueue());
} catch (YarnException e) {
LOG.info("Exception in submitting application with id " +
applicationId.getId(), e);
RMAuditLogger.logFailure(user, AuditConstants.SUBMIT_APP_REQUEST,
e.getMessage(), "ClientRMService",
- "Exception in submitting application", applicationId, callerContext);
+ "Exception in submitting application", applicationId, callerContext,
+ submissionContext.getQueue());
throw e;
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java
index 07634c7..ee1a9fd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java
@@ -39,7 +39,7 @@
static enum Keys {USER, OPERATION, TARGET, RESULT, IP, PERMISSIONS,
DESCRIPTION, APPID, APPATTEMPTID, CONTAINERID,
CALLERCONTEXT, CALLERSIGNATURE, QUEUENAME,
- INCLUDEAPPS, INCLUDECHILDQUEUES, RECURSIVE}
+ INCLUDEAPPS, INCLUDECHILDQUEUES, RECURSIVE, NODELABEL}
public static class AuditConstants {
static final String SUCCESS = "SUCCESS";
@@ -89,7 +89,7 @@ static String createSuccessLog(String user, String operation, String target,
ApplicationId appId, ApplicationAttemptId attemptId,
ContainerId containerId) {
return createSuccessLog(user, operation, target, appId, attemptId,
- containerId, null, Server.getRemoteIp());
+ containerId, null, Server.getRemoteIp(), null, null);
}
/**
@@ -115,7 +115,7 @@ private static StringBuilder createStringBuilderForSuccessEvent(String user,
static String createSuccessLog(String user, String operation, String target,
ApplicationId appId, ApplicationAttemptId attemptId,
ContainerId containerId, CallerContext callerContext,
- InetAddress ip) {
+ InetAddress ip, String queueName, String partition) {
StringBuilder b =
createStringBuilderForSuccessEvent(user, operation, target, ip);
if (appId != null) {
@@ -128,6 +128,12 @@ static String createSuccessLog(String user, String operation, String target,
add(Keys.CONTAINERID, containerId.toString(), b);
}
appendCallerContext(b, callerContext);
+ if (queueName != null) {
+ add(Keys.QUEUENAME, queueName, b);
+ }
+ if (partition != null) {
+ add(Keys.NODELABEL, partition, b);
+ }
return b.toString();
}
@@ -190,6 +196,30 @@ public static void logSuccess(String user, String operation, String target,
}
/**
+ * Create a readable and parseable audit log string for a successful event.
+ *
+ * @param user User who made the service request to the ResourceManager
+ * @param operation Operation requested by the user.
+ * @param target The target on which the operation is being performed.
+ * @param appId Application Id in which operation was performed.
+ * @param containerId Container Id in which operation was performed.
+ * @param queueName Name of queue.
+ * @param partition Name of labeled partition.
+ *
+ *
+ * Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter
+ * and hence the value fields should not contains tabs ('\t').
+ */
+ public static void logSuccess(String user, String operation, String target,
+ ApplicationId appId, ContainerId containerId, String queueName,
+ String partition) {
+ if (LOG.isInfoEnabled()) {
+ LOG.info(createSuccessLog(user, operation, target, appId, null,
+ containerId, null, null, queueName, partition));
+ }
+ }
+
+ /**
* Create a general readable and parseable audit log string for a successful
* event.
*
@@ -255,7 +285,15 @@ public static void logSuccess(String user, String operation, String target,
ApplicationId appId, CallerContext callerContext) {
if (LOG.isInfoEnabled()) {
LOG.info(createSuccessLog(user, operation, target, appId, null, null,
- callerContext, Server.getRemoteIp()));
+ callerContext, Server.getRemoteIp(), null, null));
+ }
+ }
+
+ public static void logSuccess(String user, String operation, String target,
+ ApplicationId appId, CallerContext callerContext, String queueName) {
+ if (LOG.isInfoEnabled()) {
+ LOG.info(createSuccessLog(user, operation, target, appId, null, null,
+ callerContext, Server.getRemoteIp(), queueName, null));
}
}
@@ -283,7 +321,7 @@ public static void logSuccess(String user, String operation, String target,
ApplicationId appId, InetAddress ip) {
if (LOG.isInfoEnabled()) {
LOG.info(createSuccessLog(user, operation, target, appId, null, null,
- null, ip));
+ null, ip, null, null));
}
}
@@ -342,7 +380,7 @@ private static StringBuilder createStringBuilderForFailureLog(String user,
static String createFailureLog(String user, String operation, String perm,
String target, String description, ApplicationId appId,
ApplicationAttemptId attemptId, ContainerId containerId,
- CallerContext callerContext) {
+ CallerContext callerContext, String queueName) {
StringBuilder b = createStringBuilderForFailureLog(user,
operation, target, description, perm);
if (appId != null) {
@@ -355,6 +393,9 @@ static String createFailureLog(String user, String operation, String perm,
add(Keys.CONTAINERID, containerId.toString(), b);
}
appendCallerContext(b, callerContext);
+ if (queueName != null) {
+ add(Keys.QUEUENAME, queueName, b);
+ }
return b.toString();
}
@@ -365,7 +406,7 @@ static String createFailureLog(String user, String operation, String perm,
String target, String description, ApplicationId appId,
ApplicationAttemptId attemptId, ContainerId containerId) {
return createFailureLog(user, operation, perm, target, description, appId,
- attemptId, containerId, null);
+ attemptId, containerId, null, null);
}
/**
@@ -436,7 +477,16 @@ public static void logFailure(String user, String operation, String perm,
CallerContext callerContext) {
if (LOG.isWarnEnabled()) {
LOG.warn(createFailureLog(user, operation, perm, target, description,
- appId, null, null, callerContext));
+ appId, null, null, callerContext, null));
+ }
+ }
+
+ public static void logFailure(String user, String operation, String perm,
+ String target, String description, ApplicationId appId,
+ CallerContext callerContext, String queueName) {
+ if (LOG.isWarnEnabled()) {
+ LOG.warn(createFailureLog(user, operation, perm, target, description,
+ appId, null, null, callerContext, queueName));
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
index 782ad38..e817426 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -166,9 +166,16 @@ public synchronized boolean containerCompleted(RMContainer rmContainer,
containersToPreempt.remove(containerId);
+ // In order to save space in the audit log, only include the partition
+ // if it is not the default partition.
+ String containerPartition = null;
+ if (appAMNodePartitionName != null &&
+ !appAMNodePartitionName.isEmpty()) {
+ containerPartition = appAMNodePartitionName;
+ }
RMAuditLogger.logSuccess(getUser(),
AuditConstants.RELEASE_CONTAINER, "SchedulerApp",
- getApplicationId(), containerId);
+ getApplicationId(), containerId, getQueueName(), containerPartition);
// Update usage metrics
Resource containerResource = rmContainer.getContainer().getResource();
@@ -230,9 +237,16 @@ public synchronized RMContainer allocate(NodeType type, FiCaSchedulerNode node,
+ " container=" + containerId + " host="
+ container.getNodeId().getHost() + " type=" + type);
}
+ // In order to save space in the audit log, only include the partition
+ // if it is not the default partition.
+ String partition = null;
+ if (appAMNodePartitionName != null &&
+ !appAMNodePartitionName.isEmpty()) {
+ partition = appAMNodePartitionName;
+ }
RMAuditLogger.logSuccess(getUser(),
AuditConstants.ALLOC_CONTAINER, "SchedulerApp",
- getApplicationId(), containerId);
+ getApplicationId(), containerId, getQueueName(), partition);
return rmContainer;
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index fa57dbe..fb93285 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -150,7 +150,7 @@ synchronized public void containerCompleted(RMContainer rmContainer,
RMAuditLogger.logSuccess(getUser(),
AuditConstants.RELEASE_CONTAINER, "SchedulerApp",
- getApplicationId(), containerId);
+ getApplicationId(), containerId, rmContainer.getQueueName(), null);
// Update usage metrics
Resource containerResource = rmContainer.getContainer().getResource();
@@ -404,7 +404,7 @@ else if (allowed.equals(NodeType.RACK_LOCAL) &&
}
RMAuditLogger.logSuccess(getUser(),
AuditConstants.ALLOC_CONTAINER, "SchedulerApp",
- getApplicationId(), container.getId());
+ getApplicationId(), container.getId(), getQueueName(), null);
return rmContainer;
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java
index 7ce7a49..c1d2967 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java
@@ -65,6 +65,7 @@
private static final ContainerId CONTAINERID = mock(ContainerId.class);
private static final String CALLER_CONTEXT = "context";
private static final byte[] CALLER_SIGNATURE = "signature".getBytes();
+ private static final String PARTITION = "label1";
@Before
public void setUp() throws Exception {
@@ -128,6 +129,14 @@ private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId,
ApplicationAttemptId attemptId, ContainerId containerId,
CallerContext callerContext, InetAddress remoteIp,
RMAuditLogger.ArgsBuilder args) {
+ testSuccessLogFormatHelper(checkIP, appId, attemptId, containerId,
+ callerContext, remoteIp, args, null, null);
+ }
+
+ private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId,
+ ApplicationAttemptId attemptId, ContainerId containerId,
+ CallerContext callerContext, InetAddress remoteIp,
+ RMAuditLogger.ArgsBuilder args, String queueName, String partition) {
String sLog;
InetAddress tmpIp = checkIP ? remoteIp : null;
if (args != null) {
@@ -135,7 +144,7 @@ private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId,
tmpIp, args);
} else {
sLog = RMAuditLogger.createSuccessLog(USER, OPERATION, TARGET, appId,
- attemptId, containerId, callerContext, tmpIp);
+ attemptId, containerId, callerContext, tmpIp, queueName, partition);
}
StringBuilder expLog = new StringBuilder();
expLog.append("USER=test\t");
@@ -170,6 +179,13 @@ private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId,
if (args != null) {
expLog.append("\tQUEUENAME=root");
expLog.append("\tRECURSIVE=true");
+ } else {
+ if (queueName != null) {
+ expLog.append("\tQUEUENAME=" + QUEUE);
+ }
+ }
+ if (partition != null) {
+ expLog.append("\tNODELABEL=" + PARTITION);
}
assertEquals(expLog.toString(), sLog);
}
@@ -251,6 +267,8 @@ private void testSuccessLogFormat(boolean checkIP) {
.append(Keys.QUEUENAME, QUEUE).append(Keys.RECURSIVE, "true");
testSuccessLogFormatHelper(checkIP, null, null, null, null,
Server.getRemoteIp(), args);
+ testSuccessLogFormatHelper(checkIP, null, null, null, null,
+ Server.getRemoteIp(), null, QUEUE, PARTITION);
testSuccessLogFormatHelperWithIP(checkIP, APPID, ATTEMPTID, CONTAINERID);
testSuccessLogNulls(checkIP);
}
@@ -275,7 +293,7 @@ private void testFailureLogFormatHelper(boolean checkIP, ApplicationId appId,
CallerContext callerContext, RMAuditLogger.ArgsBuilder args) {
String fLog = args == null ?
RMAuditLogger.createFailureLog(USER, OPERATION, PERM, TARGET, DESC,
- appId, attemptId, containerId, callerContext) :
+ appId, attemptId, containerId, callerContext, null) :
RMAuditLogger.createFailureLog(USER, OPERATION, PERM, TARGET, DESC,
args);
StringBuilder expLog = new StringBuilder();