diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index a0c3db6..abfb36d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -25,6 +25,7 @@ import java.nio.ByteBuffer; import java.security.AccessControlException; import java.text.MessageFormat; +import java.util.AbstractMap; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -925,6 +926,8 @@ public void remove() { callerUGI.getUserName(), allowAccess)); } + RMAuditLogger.logSuccess(callerUGI.getUserName(), + AuditConstants.GET_APPLICATIONS_REQUEST, "ClientRMService", null); GetApplicationsResponse response = recordFactory.newRecordInstance(GetApplicationsResponse.class); response.setApplicationList(reports); @@ -975,6 +978,18 @@ public GetQueueInfoResponse getQueueInfo(GetQueueInfoRequest request) GetQueueInfoResponse response = recordFactory.newRecordInstance(GetQueueInfoResponse.class); + // Use List to preserve order. + List> arguments = + new ArrayList>(); + arguments.add( + new AbstractMap.SimpleEntry("includeApplications", + String.valueOf(request.getIncludeApplications()))); + arguments.add( + new AbstractMap.SimpleEntry("includeChildQueues", + String.valueOf(request.getIncludeChildQueues()))); + arguments.add( + new AbstractMap.SimpleEntry("recursive", + String.valueOf(request.getRecursive()))); try { QueueInfo queueInfo = scheduler.getQueueInfo(request.getQueueName(), @@ -1001,8 +1016,14 @@ public GetQueueInfoResponse getQueueInfo(GetQueueInfoRequest request) } queueInfo.setApplications(appReports); response.setQueueInfo(queueInfo); + RMAuditLogger.logSuccess(callerUGI.getUserName(), + AuditConstants.GET_QUEUE_INFO_REQUEST, + "ClientRMService", request.getQueueName(), arguments); } catch (IOException ioe) { LOG.info("Failed to getQueueInfo for " + request.getQueueName(), ioe); + RMAuditLogger.logFailure(callerUGI.getUserName(), + AuditConstants.GET_QUEUE_INFO_REQUEST, "UNKNOWN", "ClientRMService", + ioe.getMessage(), request.getQueueName(), arguments); } return response; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java index 29ce636..22f2f41 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java @@ -19,6 +19,8 @@ import java.io.UnsupportedEncodingException; import java.net.InetAddress; +import java.util.List; +import java.util.Map; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -39,7 +41,7 @@ enum Keys {USER, OPERATION, TARGET, RESULT, IP, PERMISSIONS, DESCRIPTION, APPID, APPATTEMPTID, CONTAINERID, - CALLERCONTEXT, CALLERSIGNATURE, RESOURCE} + CALLERCONTEXT, CALLERSIGNATURE, RESOURCE, QUEUE} public static class AuditConstants { static final String SUCCESS = "SUCCESS"; @@ -59,6 +61,10 @@ = "Get Application Attempt Report"; public static final String GET_CONTAINERS = "Get Containers"; public static final String GET_CONTAINER_REPORT = "Get Container Report"; + public static final String GET_QUEUE_INFO_REQUEST = + "Get Queue Info Request"; + public static final String GET_APPLICATIONS_REQUEST = + "Get Applications Request"; public static final String FINISH_SUCCESS_APP = "Application Finished - Succeeded"; public static final String FINISH_FAILED_APP = "Application Finished - Failed"; public static final String FINISH_KILLED_APP = "Application Finished - Killed"; @@ -97,12 +103,11 @@ static String createSuccessLog(String user, String operation, String target, } /** - * A helper api for creating an audit log for a successful event. + * A helper function for creating an string builder. + * It builds common message for a success event. */ - static String createSuccessLog(String user, String operation, String target, - ApplicationId appId, ApplicationAttemptId attemptId, - ContainerId containerId, Resource resource, CallerContext callerContext, - InetAddress ip) { + private static StringBuilder createStringBuilderForSuccessEvent(String user, + String operation, String target, InetAddress ip) { StringBuilder b = new StringBuilder(); start(Keys.USER, user, b); if (ip != null) { @@ -111,6 +116,18 @@ static String createSuccessLog(String user, String operation, String target, add(Keys.OPERATION, operation, b); add(Keys.TARGET, target ,b); add(Keys.RESULT, AuditConstants.SUCCESS, b); + return b; + } + + /** + * A helper api for creating an audit log for a successful event. + */ + static String createSuccessLog(String user, String operation, String target, + ApplicationId appId, ApplicationAttemptId attemptId, + ContainerId containerId, Resource resource, CallerContext callerContext, + InetAddress ip) { + StringBuilder b = + createStringBuilderForSuccessEvent(user, operation, target, ip); if (appId != null) { add(Keys.APPID, appId.toString(), b); } @@ -151,6 +168,26 @@ private static void appendCallerContext(StringBuilder sb, CallerContext callerCo } /** + * A helper api for creating an audit log for a successful event. + */ + @SuppressWarnings("rawtypes") + static String createSuccessLog(String user, String operation, String target, + String queue, List> args) { + StringBuilder b = + createStringBuilderForSuccessEvent(user, operation, target, + Server.getRemoteIp()); + if(queue != null) { + add(Keys.QUEUE, queue, b); + } + if(args != null && (!args.isEmpty())) { + for (Map.Entry entry : args) { + add((String)entry.getKey(), (String)entry.getValue(), b); + } + } + return b.toString(); + } + + /** * Create a readable and parseable audit log string for a successful event. * * @param user User who made the service request to the ResourceManager @@ -178,6 +215,26 @@ public static void logSuccess(String user, String operation, String target, * @param user User who made the service request to the ResourceManager. * @param operation Operation requested by the user. * @param target The target on which the operation is being performed. + * @param queueName The queue on which the operation requests. + * @param args The arguments for the operation request. + * + *

+ * Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter + * and hence the value fields should not contains tabs ('\t'). + */ + public static void logSuccess(String user, String operation, String target, + String queueName, List> args) { + if (LOG.isInfoEnabled()) { + LOG.info(createSuccessLog(user, operation, target, queueName, args)); + } + } + + /** + * Create a readable and parseable audit log string for a successful event. + * + * @param user User who made the service request to the ResourceManager. + * @param operation Operation requested by the user. + * @param target The target on which the operation is being performed. * @param appId Application Id in which operation was performed. * @param attemptId Application Attempt Id in which operation was performed. * @@ -265,10 +322,8 @@ public static void logSuccess(String user, String operation, String target) { } } - static String createFailureLog(String user, String operation, String perm, - String target, String description, ApplicationId appId, - ApplicationAttemptId attemptId, ContainerId containerId, - Resource resource, CallerContext callerContext) { + private static StringBuilder createStringBuilderForFailureLog(String user, + String operation, String target, String description, String perm) { StringBuilder b = new StringBuilder(); start(Keys.USER, user, b); addRemoteIP(b); @@ -277,6 +332,18 @@ static String createFailureLog(String user, String operation, String perm, add(Keys.RESULT, AuditConstants.FAILURE, b); add(Keys.DESCRIPTION, description, b); add(Keys.PERMISSIONS, perm, b); + return b; + } + + /** + * A helper api for creating an audit log for a failure event. + */ + static String createFailureLog(String user, String operation, String perm, + String target, String description, ApplicationId appId, + ApplicationAttemptId attemptId, ContainerId containerId, + Resource resource, CallerContext callerContext) { + StringBuilder b = createStringBuilderForFailureLog(user, + operation, target, description, perm); if (appId != null) { add(Keys.APPID, appId.toString(), b); } @@ -304,6 +371,26 @@ static String createFailureLog(String user, String operation, String perm, } /** + * A helper api for creating an audit log for a failure event. + */ + @SuppressWarnings("rawtypes") + static String createFailureLog(String user, String operation, String perm, + String target, String description, + String queueName, List> args) { + StringBuilder b = createStringBuilderForFailureLog(user, + operation, target, description, perm); + if(queueName != null) { + add(Keys.QUEUE, queueName, b); + } + if(args != null && (!args.isEmpty())) { + for (Map.Entry entry : args) { + add((String)entry.getKey(), (String)entry.getValue(), b); + } + } + return b.toString(); + } + + /** * Create a readable and parseable audit log string for a failed event. * * @param user User who made the service request. @@ -408,7 +495,30 @@ public static void logFailure(String user, String operation, String perm, } /** - * A helper api to add remote IP address + * Create a readable and parseable audit log string for a failed event. + * + * @param user User who made the service request. + * @param operation Operation requested by the user. + * @param perm Target permissions. + * @param target The target on which the operation is being performed. + * @param queueName The queue on which the operation requests. + * @param args The arguments for the operation request. + * + *

+ * Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter + * and hence the value fields should not contains tabs ('\t'). + */ + public static void logFailure(String user, String operation, + String perm, String target, String description, + String queueName, List> args) { + if (LOG.isWarnEnabled()) { + LOG.warn(createFailureLog(user, operation, perm, + target, description,queueName, args)); + } + } + + /** + * A helper api to add remote IP address. */ static void addRemoteIP(StringBuilder b) { InetAddress ip = Server.getRemoteIp(); @@ -434,4 +544,13 @@ static void add(Keys key, String value, StringBuilder b) { b.append(AuditConstants.PAIR_SEPARATOR).append(key.name()) .append(AuditConstants.KEY_VAL_SEPARATOR).append(value); } + + /** + * Appends the key-val pair to the passed builder in the following format + * key=value + */ + static void add(String key, String value, StringBuilder b) { + b.append(AuditConstants.PAIR_SEPARATOR).append(key) + .append(AuditConstants.KEY_VAL_SEPARATOR).append(value); + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java index 3311f92..4d73b47 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java @@ -24,6 +24,10 @@ import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.UnknownHostException; +import java.util.AbstractMap; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; import com.google.protobuf.BlockingService; import com.google.protobuf.RpcController; @@ -60,6 +64,7 @@ private static final String TARGET = "tgt"; private static final String PERM = "admin group"; private static final String DESC = "description of an audit log"; + private static final String QUEUE = "root"; private static final ApplicationId APPID = mock(ApplicationId.class); private static final ApplicationAttemptId ATTEMPTID = mock(ApplicationAttemptId.class); private static final ContainerId CONTAINERID = mock(ContainerId.class); @@ -114,7 +119,7 @@ private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId, ApplicationAttemptId attemptId, ContainerId containerId, CallerContext callerContext, Resource resource) { testSuccessLogFormatHelper(checkIP, appId, attemptId, containerId, - callerContext, resource, Server.getRemoteIp()); + callerContext, resource, Server.getRemoteIp(), null, null); } /** @@ -122,15 +127,20 @@ private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId, */ private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId, ApplicationAttemptId attemptId, ContainerId containerId, - CallerContext callerContext, Resource resource, InetAddress remoteIp) { - + CallerContext callerContext, Resource resource, InetAddress remoteIp, + String queue, List> args) { String sLog; - if (checkIP) { - sLog = RMAuditLogger.createSuccessLog(USER, OPERATION, TARGET, appId, - attemptId, containerId, resource, callerContext, remoteIp); + if (queue == null) { + if (checkIP) { + sLog = RMAuditLogger.createSuccessLog(USER, OPERATION, TARGET, appId, + attemptId, containerId, resource, callerContext, remoteIp); + } else { + sLog = RMAuditLogger.createSuccessLog(USER, OPERATION, TARGET, appId, + attemptId, containerId, resource, callerContext, null); + } } else { - sLog = RMAuditLogger.createSuccessLog(USER, OPERATION, TARGET, appId, - attemptId, containerId, resource, callerContext, null); + sLog = + RMAuditLogger.createSuccessLog(USER, OPERATION, TARGET, QUEUE, args); } StringBuilder expLog = new StringBuilder(); expLog.append("USER=test\t"); @@ -165,6 +175,12 @@ private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId, expLog.append("\tCALLERSIGNATURE=signature"); } } + if (queue != null) { + expLog.append("\tQUEUE=root"); + } + if (args != null) { + expLog.append("\trecursive=true"); + } assertEquals(expLog.toString(), sLog); } @@ -172,7 +188,7 @@ private void testSuccessLogFormatHelperWithIP(boolean checkIP, ApplicationId appId, ApplicationAttemptId attemptId, ContainerId containerId, InetAddress ip) { testSuccessLogFormatHelper(checkIP, appId, attemptId, containerId, null, - null, ip); + null, ip, null, null); } /** @@ -241,13 +257,19 @@ private void testSuccessLogFormat(boolean checkIP) { testSuccessLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID, new CallerContext.Builder(CALLER_CONTEXT).setSignature(CALLER_SIGNATURE) .build(), RESOURCE); + List> args = + new ArrayList>(); + args.add(new AbstractMap.SimpleEntry("recursive", "true")); + testSuccessLogFormatHelper(checkIP, null, null, null, null, null, null, + QUEUE, args); testSuccessLogFormatHelperWithIP(checkIP, APPID, ATTEMPTID, CONTAINERID); testSuccessLogNulls(checkIP); } private void testFailureLogFormatHelper(boolean checkIP, ApplicationId appId, ApplicationAttemptId attemptId, ContainerId containerId) { - testFailureLogFormatHelper(checkIP, appId, attemptId, containerId, null, null); + testFailureLogFormatHelper(checkIP, appId, attemptId, containerId, null, + null, null, null); } /** @@ -255,10 +277,13 @@ private void testFailureLogFormatHelper(boolean checkIP, ApplicationId appId, */ private void testFailureLogFormatHelper(boolean checkIP, ApplicationId appId, ApplicationAttemptId attemptId, ContainerId containerId, - CallerContext callerContext, Resource resource) { - String fLog = + CallerContext callerContext, Resource resource, String queue, + List> args) { + String fLog = queue == null ? RMAuditLogger.createFailureLog(USER, OPERATION, PERM, TARGET, DESC, - appId, attemptId, containerId, resource, callerContext); + appId, attemptId, containerId, resource, callerContext) : + RMAuditLogger.createFailureLog(USER, OPERATION, PERM, TARGET, DESC, + QUEUE, args); StringBuilder expLog = new StringBuilder(); expLog.append("USER=test\t"); if (checkIP) { @@ -288,6 +313,12 @@ private void testFailureLogFormatHelper(boolean checkIP, ApplicationId appId, expLog.append("\tCALLERSIGNATURE=signature"); } } + if (queue != null) { + expLog.append("\tQUEUE=root"); + } + if (args != null) { + expLog.append("\trecursive=true"); + } assertEquals(expLog.toString(), fLog); } @@ -306,14 +337,22 @@ private void testFailureLogFormat(boolean checkIP) { testFailureLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID); testFailureLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID, - new CallerContext.Builder(null).setSignature(null).build(), RESOURCE); + new CallerContext.Builder(null).setSignature(null).build(), RESOURCE, + null, null); testFailureLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID, - new CallerContext.Builder(CALLER_CONTEXT).setSignature(null).build(), RESOURCE); + new CallerContext.Builder(CALLER_CONTEXT).setSignature(null).build(), + RESOURCE, null, null); testFailureLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID, - new CallerContext.Builder(null).setSignature(CALLER_SIGNATURE).build(), RESOURCE); + new CallerContext.Builder(null).setSignature(CALLER_SIGNATURE).build(), + RESOURCE, null, null); testFailureLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID, new CallerContext.Builder(CALLER_CONTEXT).setSignature(CALLER_SIGNATURE) - .build(), RESOURCE); + .build(), RESOURCE, null, null); + List> args = + new ArrayList>(); + args.add(new AbstractMap.SimpleEntry("recursive", "true")); + testFailureLogFormatHelper(checkIP, null, null, null, null, null, QUEUE, + args); } /**