diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index 812267d..aaa266d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -27,6 +27,7 @@ import java.util.Collection; import java.util.Collections; import java.util.EnumSet; +import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -903,7 +904,8 @@ public void remove() { reports.add(application.createAndGetApplicationReport( callerUGI.getUserName(), allowAccess)); } - + RMAuditLogger.logSuccess(callerUGI.getUserName(), + AuditConstants.GET_APPLICATIONS_REQUEST, "ClientRMService"); GetApplicationsResponse response = recordFactory.newRecordInstance(GetApplicationsResponse.class); response.setApplicationList(reports); @@ -943,6 +945,13 @@ public GetQueueInfoResponse getQueueInfo(GetQueueInfoRequest request) GetQueueInfoResponse response = recordFactory.newRecordInstance(GetQueueInfoResponse.class); + Map arguments = new HashMap(); + arguments.put("includeApplications", + String.valueOf(request.getIncludeApplications())); + arguments.put("includeChildQueues", + String.valueOf(request.getIncludeChildQueues())); + arguments.put("recursive", + String.valueOf(request.getRecursive())); try { QueueInfo queueInfo = scheduler.getQueueInfo(request.getQueueName(), @@ -969,8 +978,14 @@ public GetQueueInfoResponse getQueueInfo(GetQueueInfoRequest request) } queueInfo.setApplications(appReports); response.setQueueInfo(queueInfo); + RMAuditLogger.logSuccess(callerUGI.getUserName(), + AuditConstants.GET_QUEUE_INFO_REQUEST, + "ClientRMService", request.getQueueName(), arguments); } catch (IOException ioe) { LOG.info("Failed to getQueueInfo for " + request.getQueueName(), ioe); + RMAuditLogger.logFailure(callerUGI.getUserName(), + AuditConstants.GET_QUEUE_INFO_REQUEST, "UNKNOWN", "ClientRMService", + ioe.getMessage(), request.getQueueName(), arguments); } return response; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java index 8cb7c80..ef1a86d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.resourcemanager; import java.net.InetAddress; +import java.util.Map; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -34,8 +35,10 @@ public class RMAuditLogger { private static final Log LOG = LogFactory.getLog(RMAuditLogger.class); - static enum Keys {USER, OPERATION, TARGET, RESULT, IP, PERMISSIONS, - DESCRIPTION, APPID, APPATTEMPTID, CONTAINERID} + static enum Keys { + USER, OPERATION, TARGET, RESULT, IP, PERMISSIONS, + DESCRIPTION, APPID, APPATTEMPTID, CONTAINERID, QUEUE + } public static class AuditConstants { static final String SUCCESS = "SUCCESS"; @@ -47,6 +50,10 @@ public static final String KILL_APP_REQUEST = "Kill Application Request"; public static final String SUBMIT_APP_REQUEST = "Submit Application Request"; public static final String MOVE_APP_REQUEST = "Move Application Request"; + public static final String GET_QUEUE_INFO_REQUEST = + "All Enable Get Queue Info Request"; + public static final String GET_APPLICATIONS_REQUEST = + "Get Applications Request"; public static final String FINISH_SUCCESS_APP = "Application Finished - Succeeded"; public static final String FINISH_FAILED_APP = "Application Finished - Failed"; public static final String FINISH_KILLED_APP = "Application Finished - Killed"; @@ -71,16 +78,28 @@ } /** - * A helper api for creating an audit log for a successful event. + * A helper function for creating an string builder. + * It builds common message for a success event. */ - static String createSuccessLog(String user, String operation, String target, - ApplicationId appId, ApplicationAttemptId attemptId, ContainerId containerId) { + private static StringBuilder createStringBuilderForSuccessEvent(String user, + String operation, String target) { StringBuilder b = new StringBuilder(); start(Keys.USER, user, b); addRemoteIP(b); add(Keys.OPERATION, operation, b); - add(Keys.TARGET, target ,b); + add(Keys.TARGET, target, b); add(Keys.RESULT, AuditConstants.SUCCESS, b); + return b; + } + + /** + * A helper api for creating an audit log for a successful event. + */ + static String createSuccessLog(String user, String operation, String target, + ApplicationId appId, ApplicationAttemptId attemptId, + ContainerId containerId) { + StringBuilder b = + createStringBuilderForSuccessEvent(user, operation, target); if (appId != null) { add(Keys.APPID, appId.toString(), b); } @@ -94,6 +113,25 @@ static String createSuccessLog(String user, String operation, String target, } /** + * A helper api for creating an audit log for a successful event. + */ + @SuppressWarnings("rawtypes") + static String createSuccessLog(String user, String operation, String target, + String queue, Map args) { + StringBuilder b = + createStringBuilderForSuccessEvent(user, operation, target); + if(queue != null) { + add(Keys.QUEUE, queue, b); + } + if(args != null && (!args.isEmpty())) { + for (Map.Entry entry : args.entrySet()) { + add((String)entry.getKey(), (String)entry.getValue(), b); + } + } + return b.toString(); + } + + /** * Create a readable and parseable audit log string for a successful event. * * @param user User who made the service request to the ResourceManager @@ -120,6 +158,26 @@ public static void logSuccess(String user, String operation, String target, * @param user User who made the service request to the ResourceManager. * @param operation Operation requested by the user. * @param target The target on which the operation is being performed. + * @param queueName The queue on which the operation requests. + * @param args The arguments for the operation request. + * + *

+ * Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter + * and hence the value fields should not contains tabs ('\t'). + */ + public static void logSuccess(String user, String operation, String target, + String queueName, Map args) { + if (LOG.isInfoEnabled()) { + LOG.info(createSuccessLog(user, operation, target, queueName, args)); + } + } + + /** + * Create a readable and parseable audit log string for a successful event. + * + * @param user User who made the service request to the ResourceManager. + * @param operation Operation requested by the user. + * @param target The target on which the operation is being performed. * @param appId Application Id in which operation was performed. * @param attemptId Application Attempt Id in which operation was performed. * @@ -173,11 +231,11 @@ public static void logSuccess(String user, String operation, String target) { } /** - * A helper api for creating an audit log for a failure event. + * A helper function for creating an string builder. + * It builds common message for a failure event. */ - static String createFailureLog(String user, String operation, String perm, - String target, String description, ApplicationId appId, - ApplicationAttemptId attemptId, ContainerId containerId) { + private static StringBuilder createStringBuilderForFailureLog(String user, + String operation, String target, String description, String perm) { StringBuilder b = new StringBuilder(); start(Keys.USER, user, b); addRemoteIP(b); @@ -186,6 +244,17 @@ static String createFailureLog(String user, String operation, String perm, add(Keys.RESULT, AuditConstants.FAILURE, b); add(Keys.DESCRIPTION, description, b); add(Keys.PERMISSIONS, perm, b); + return b; + } + + /** + * A helper api for creating an audit log for a failure event. + */ + static String createFailureLog(String user, String operation, String perm, + String target, String description, ApplicationId appId, + ApplicationAttemptId attemptId, ContainerId containerId) { + StringBuilder b = createStringBuilderForFailureLog(user, + operation, target, description, perm); if (appId != null) { add(Keys.APPID, appId.toString(), b); } @@ -199,6 +268,26 @@ static String createFailureLog(String user, String operation, String perm, } /** + * A helper api for creating an audit log for a failure event. + */ + @SuppressWarnings("rawtypes") + static String createFailureLog(String user, String operation, String perm, + String target, String description, + String queueName, Map args) { + StringBuilder b = createStringBuilderForFailureLog(user, + operation, target, description, perm); + if(queueName != null) { + add(Keys.QUEUE, queueName, b); + } + if(args != null && (!args.isEmpty())) { + for (Map.Entry entry : args.entrySet()) { + add((String)entry.getKey(), (String)entry.getValue(), b); + } + } + return b.toString(); + } + + /** * Create a readable and parseable audit log string for a failed event. * * @param user User who made the service request. @@ -294,7 +383,30 @@ public static void logFailure(String user, String operation, String perm, } /** - * A helper api to add remote IP address + * Create a readable and parseable audit log string for a failed event. + * + * @param user User who made the service request. + * @param operation Operation requested by the user. + * @param perm Target permissions. + * @param target The target on which the operation is being performed. + * @param queueName The queue on which the operation requests. + * @param args The arguments for the operation request. + * + *

+ * Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter + * and hence the value fields should not contains tabs ('\t'). + */ + public static void logFailure(String user, String operation, + String perm, String target, String description, + String queueName, Map args) { + if (LOG.isWarnEnabled()) { + LOG.warn(createFailureLog(user, operation, perm, + target, description,queueName, args)); + } + } + + /** + * A helper api to add remote IP address. */ static void addRemoteIP(StringBuilder b) { InetAddress ip = Server.getRemoteIp(); @@ -320,4 +432,13 @@ static void add(Keys key, String value, StringBuilder b) { b.append(AuditConstants.PAIR_SEPARATOR).append(key.name()) .append(AuditConstants.KEY_VAL_SEPARATOR).append(value); } + + /** + * Appends the key-val pair to the passed builder in the following format + * key=value + */ + static void add(String key, String value, StringBuilder b) { + b.append(AuditConstants.PAIR_SEPARATOR).append(key) + .append(AuditConstants.KEY_VAL_SEPARATOR).append(value); + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java index 41776cc..0448b83 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java @@ -23,6 +23,8 @@ import java.net.InetAddress; import java.net.InetSocketAddress; +import java.util.HashMap; +import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.ipc.RPC; @@ -47,6 +49,7 @@ private static final String TARGET = "tgt"; private static final String PERM = "admin group"; private static final String DESC = "description of an audit log"; + private static final String QUEUE = "root"; private static final ApplicationId APPID = mock(ApplicationId.class); private static final ApplicationAttemptId ATTEMPTID = mock(ApplicationAttemptId.class); private static final ContainerId CONTAINERID = mock(ContainerId.class); @@ -87,14 +90,17 @@ public void testKeyValLogFormat() throws Exception { assertEquals(expLog.toString(), actLog.toString()); } - /** * Test the AuditLog format for successful events. */ private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId, - ApplicationAttemptId attemptId, ContainerId containerId) { - String sLog = RMAuditLogger.createSuccessLog(USER, OPERATION, TARGET, - appId, attemptId, containerId); + ApplicationAttemptId attemptId, ContainerId containerId, String queue, + Map args) { + String sLog = queue == null ? + RMAuditLogger.createSuccessLog(USER, OPERATION, TARGET, + appId, attemptId, containerId) : + RMAuditLogger.createSuccessLog(USER, + OPERATION, TARGET, QUEUE, args); StringBuilder expLog = new StringBuilder(); expLog.append("USER=test\t"); if (checkIP) { @@ -112,6 +118,12 @@ private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId, if (containerId != null) { expLog.append("\tCONTAINERID=container_1"); } + if (queue != null) { + expLog.append("\tQUEUE=root"); + } + if (args != null) { + expLog.append("\trecursive=true"); + } assertEquals(expLog.toString(), sLog); } @@ -136,15 +148,19 @@ private void testSuccessLogNulls(boolean checkIP) { * parameters. */ private void testSuccessLogFormat(boolean checkIP) { - testSuccessLogFormatHelper(checkIP, null, null, null); - testSuccessLogFormatHelper(checkIP, APPID, null, null); - testSuccessLogFormatHelper(checkIP, null, null, CONTAINERID); - testSuccessLogFormatHelper(checkIP, null, ATTEMPTID, null); - testSuccessLogFormatHelper(checkIP, APPID, ATTEMPTID, null); - testSuccessLogFormatHelper(checkIP, APPID, null, CONTAINERID); - testSuccessLogFormatHelper(checkIP, null, ATTEMPTID, CONTAINERID); - testSuccessLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID); - testSuccessLogNulls(checkIP); + testSuccessLogFormatHelper(checkIP, null, null, null, null, null); + testSuccessLogFormatHelper(checkIP, APPID, null, null, null, null); + testSuccessLogFormatHelper(checkIP, null, null, CONTAINERID, null, null); + testSuccessLogFormatHelper(checkIP, null, ATTEMPTID, null, null, null); + testSuccessLogFormatHelper(checkIP, APPID, ATTEMPTID, null, null, null); + testSuccessLogFormatHelper(checkIP, APPID, null, CONTAINERID, null, null); + testSuccessLogFormatHelper(checkIP, null, ATTEMPTID, + CONTAINERID, null, null); + testSuccessLogFormatHelper(checkIP, APPID, ATTEMPTID, + CONTAINERID, null, null); + Map args = new HashMap(); + args.put("recursive", "true"); + testSuccessLogFormatHelper(checkIP, null, null, null, QUEUE, args); } @@ -152,10 +168,12 @@ private void testSuccessLogFormat(boolean checkIP) { * Test the AuditLog format for failure events. */ private void testFailureLogFormatHelper(boolean checkIP, ApplicationId appId, - ApplicationAttemptId attemptId, ContainerId containerId) { - String fLog = + ApplicationAttemptId attemptId, ContainerId containerId, String queue, + Mapargs) { + String fLog = queue == null ? RMAuditLogger.createFailureLog(USER, OPERATION, PERM, TARGET, DESC, - appId, attemptId, containerId); + appId, attemptId, containerId) : + RMAuditLogger.createFailureLog(USER, OPERATION, PERM, TARGET, DESC, QUEUE, args); StringBuilder expLog = new StringBuilder(); expLog.append("USER=test\t"); if (checkIP) { @@ -174,6 +192,12 @@ private void testFailureLogFormatHelper(boolean checkIP, ApplicationId appId, if (containerId != null) { expLog.append("\tCONTAINERID=container_1"); } + if (queue != null) { + expLog.append("\tQUEUE=root"); + } + if (args != null) { + expLog.append("\trecursive=true"); + } assertEquals(expLog.toString(), fLog); } @@ -182,14 +206,19 @@ private void testFailureLogFormatHelper(boolean checkIP, ApplicationId appId, * parameters. */ private void testFailureLogFormat(boolean checkIP) { - testFailureLogFormatHelper(checkIP, null, null, null); - testFailureLogFormatHelper(checkIP, APPID, null, null); - testFailureLogFormatHelper(checkIP, null, null, CONTAINERID); - testFailureLogFormatHelper(checkIP, null, ATTEMPTID, null); - testFailureLogFormatHelper(checkIP, APPID, ATTEMPTID, null); - testFailureLogFormatHelper(checkIP, APPID, null, CONTAINERID); - testFailureLogFormatHelper(checkIP, null, ATTEMPTID, CONTAINERID); - testFailureLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID); + testFailureLogFormatHelper(checkIP, null, null, null, null, null); + testFailureLogFormatHelper(checkIP, APPID, null, null, null, null); + testFailureLogFormatHelper(checkIP, null, null, CONTAINERID, null, null); + testFailureLogFormatHelper(checkIP, null, ATTEMPTID, null, null, null); + testFailureLogFormatHelper(checkIP, APPID, ATTEMPTID, null, null, null); + testFailureLogFormatHelper(checkIP, APPID, null, CONTAINERID, null, null); + testFailureLogFormatHelper(checkIP, null, + ATTEMPTID, CONTAINERID, null, null); + testFailureLogFormatHelper(checkIP, APPID, + ATTEMPTID, CONTAINERID, null, null); + Map args = new HashMap(); + args.put("recursive", "true"); + testFailureLogFormatHelper(checkIP, null, null, null, QUEUE, args); } /**