diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterAuditLogger.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterAuditLogger.java new file mode 100644 index 00000000000..1b9a5519168 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterAuditLogger.java @@ -0,0 +1,276 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.router; + +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Manages Router audit logs. + * + * Audit log format is written as key=value pairs. Tab separated. + */ +public class RouterAuditLogger { + private static final Logger LOG = + LoggerFactory.getLogger(RouterAuditLogger.class); + + private RouterAuditLogger() { + } + + enum Keys { + USER, OPERATION, TARGET, RESULT, IP, PERMISSIONS, DESCRIPTION, APPID, + SUBCLUSTERID } + + public static class AuditConstants { + static final String SUCCESS = "SUCCESS"; + static final String FAILURE = "FAILURE"; + static final String KEY_VAL_SEPARATOR = "="; + static final char PAIR_SEPARATOR = '\t'; + + public static final String GET_NEW_APP = "Get New App"; + public static final String SUBMIT_NEW_APP = "Submit New App"; + public static final String FORCE_KILL_APP = "Force Kill App"; + public static final String GET_APP_REPORT = "Get Application Report"; + } + + /** + * Create a readable and parseable audit log string for a successful event. + * + * @param user User who made the service request to the Router + * @param operation Operation requested by the user. + * @param target The target on which the operation is being performed. + * @param appId Application Id in which operation was performed. + * + *

+ * Note that the {@link RouterAuditLogger} uses tabs ('\t') as a key-val + * delimiter and hence the value fields should not contains tabs ('\t'). + */ + public static void logSuccess(String user, String operation, String target, + ApplicationId appId) { + if (LOG.isInfoEnabled()) { + LOG.info(createSuccessLog(user, operation, target, appId, null)); + } + } + + /** + * Create a readable and parseable audit log string for a successful event. + * + * @param user User who made the service request to the Router + * @param operation Operation requested by the user. + * @param target The target on which the operation is being performed. + * @param appId Application Id in which operation was performed. + * @param subClusterId Subcluster Id in which operation is performed. + * + *

+ * Note that the {@link RouterAuditLogger} uses tabs ('\t') as a key-val + * delimiter and hence the value fields should not contains tabs ('\t'). + */ + public static void logSuccess(String user, String operation, String target, + ApplicationId appId, SubClusterId subClusterId) { + if (LOG.isInfoEnabled()) { + LOG.info(createSuccessLog(user, operation, target, appId, subClusterId)); + } + } + + /** + * A helper api for creating an audit log for a successful event. + */ + static String createSuccessLog(String user, String operation, String target, + ApplicationId appId, SubClusterId subClusterID) { + StringBuilder b = + createStringBuilderForSuccessEvent(user, operation, target); + if (appId != null) { + add(Keys.APPID, appId.toString(), b); + } + if (subClusterID != null) { + add(Keys.SUBCLUSTERID, subClusterID.toString(), b); + } + return b.toString(); + } + + /** + * A helper function for creating the common portion of a successful + * log message. + */ + private static StringBuilder createStringBuilderForSuccessEvent(String user, + String operation, String target) { + StringBuilder b = new StringBuilder(); + start(Keys.USER, user, b); + add(Keys.OPERATION, operation, b); + add(Keys.TARGET, target, b); + add(Keys.RESULT, AuditConstants.SUCCESS, b); + return b; + } + + /** + * Create a readable and parseable audit log string for a failed event. + * + * @param user User who made the service request. + * @param operation Operation requested by the user. + * @param perm Target permissions. + * @param target The target on which the operation is being performed. + * @param description Some additional information as to why the operation + * failed. + * + *

+ * Note that the {@link RouterAuditLogger} uses tabs ('\t') as a key-val + * delimiter and hence the value fields should not contains tabs ('\t'). + */ + public static void logFailure(String user, String operation, String perm, + String target, String description) { + if (LOG.isInfoEnabled()) { + LOG.info( + createFailureLog(user, operation, perm, target, description, null, + null)); + } + } + + /** + * Create a readable and parseable audit log string for a failed event. + * + * @param user User who made the service request. + * @param operation Operation requested by the user. + * @param perm Target permissions. + * @param target The target on which the operation is being performed. + * @param description Some additional information as to why the operation + * failed. + * @param appId Application Id in which operation was performed. + * + *

+ * Note that the {@link RouterAuditLogger} uses tabs ('\t') as a key-val + * delimiter and hence the value fields should not contains tabs ('\t'). + */ + public static void logFailure(String user, String operation, String perm, + String target, String description, ApplicationId appId) { + if (LOG.isInfoEnabled()) { + LOG.info( + createFailureLog(user, operation, perm, target, description, appId, + null)); + } + } + + /** + * Create a readable and parseable audit log string for a failed event. + * + * @param user User who made the service request. + * @param operation Operation requested by the user. + * @param perm Target permissions. + * @param target The target on which the operation is being performed. + * @param description Some additional information as to why the operation + * failed. + * @param appId Application Id in which operation was performed. + * @param subClusterId SubCluster Id in which operation was performed. + * + *

+ * Note that the {@link RouterAuditLogger} uses tabs ('\t') as a key-val + * delimiter and hence the value fields should not contains tabs ('\t'). + */ + public static void logFailure(String user, String operation, String perm, + String target, String description, ApplicationId appId, + SubClusterId subClusterId) { + if (LOG.isInfoEnabled()) { + LOG.info(createFailureLog(user, operation, perm, target, description, + appId, subClusterId)); + } + } + + /** + * A helper api for creating an audit log for a failure event. + */ + static String createFailureLog(String user, String operation, String perm, + String target, String description, ApplicationId appId, + SubClusterId subClusterId) { + StringBuilder b = + createStringBuilderForFailureLog(user, operation, target, description, + perm); + if (appId != null) { + add(Keys.APPID, appId.toString(), b); + } + if (subClusterId != null) { + add(Keys.SUBCLUSTERID, subClusterId.toString(), b); + } + return b.toString(); + } + + /** + * A helper function for creating the common portion of a failure + * log message. + */ + private static StringBuilder createStringBuilderForFailureLog(String user, + String operation, String target, String description, String perm) { + StringBuilder b = new StringBuilder(); + start(Keys.USER, user, b); + add(Keys.OPERATION, operation, b); + add(Keys.TARGET, target, b); + add(Keys.RESULT, AuditConstants.FAILURE, b); + add(Keys.DESCRIPTION, description, b); + add(Keys.PERMISSIONS, perm, b); + return b; + } + + /** + * Adds the first key-val pair to the passed builder in the following format + * key=value. + */ + static void start(Keys key, String value, StringBuilder b) { + b.append(key.name()).append(AuditConstants.KEY_VAL_SEPARATOR).append(value); + } + + /** + * Appends the key-val pair to the passed builder in the following format + * key=value. + */ + static void add(Keys key, String value, StringBuilder b) { + b.append(AuditConstants.PAIR_SEPARATOR).append(key.name()) + .append(AuditConstants.KEY_VAL_SEPARATOR).append(value); + } + + /** + * Appends the key-val pair to the passed builder in the following format + * key=value. + */ + static void add(ArgsBuilder args, StringBuilder b) { + b.append(AuditConstants.PAIR_SEPARATOR).append(args.getArgs()); + } + + /** + * Builder to create and pass a list of arbitrary key value pairs for logging. + */ + public static class ArgsBuilder { + private StringBuilder b; + + public ArgsBuilder() { + b = new StringBuilder(); + } + + public ArgsBuilder append(Keys key, String value) { + if (b.length() != 0) { + b.append(AuditConstants.PAIR_SEPARATOR); + } + b.append(key.name()).append(AuditConstants.KEY_VAL_SEPARATOR) + .append(value); + return this; + } + + public StringBuilder getArgs() { + return b; + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java index 7e8e7af3c7a..614471d4f36 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java @@ -123,6 +123,7 @@ import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo; import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade; +import org.apache.hadoop.yarn.server.router.RouterAuditLogger; import org.apache.hadoop.yarn.server.router.RouterMetrics; import org.apache.hadoop.yarn.server.router.RouterServerUtil; import org.apache.hadoop.yarn.util.Clock; @@ -293,6 +294,9 @@ public GetNewApplicationResponse getNewApplication( long stopTime = clock.getTime(); routerMetrics.succeededAppsCreated(stopTime - startTime); + RouterAuditLogger.logSuccess(user.toString(), + RouterAuditLogger.AuditConstants.GET_NEW_APP, + "RouterClientRMService", response.getApplicationId()); return response; } else { // Empty response from the ResourceManager. @@ -304,7 +308,9 @@ public GetNewApplicationResponse getNewApplication( routerMetrics.incrAppsFailedCreated(); String errMsg = "Fail to create a new application."; - LOG.error(errMsg); + RouterAuditLogger.logFailure(user.toString(), + RouterAuditLogger.AuditConstants.GET_NEW_APP, "UNKNOWN", + "RouterClientRMService", errMsg); throw new YarnException(errMsg); } @@ -383,9 +389,13 @@ public SubmitApplicationResponse submitApplication( || request.getApplicationSubmissionContext() .getApplicationId() == null) { routerMetrics.incrAppsFailedSubmitted(); - RouterServerUtil - .logAndThrowException("Missing submitApplication request or " - + "applicationSubmissionContex information.", null); + String errMsg = + "Missing submitApplication request or applicationSubmissionContex " + + "information."; + RouterAuditLogger.logFailure(user.toString(), + RouterAuditLogger.AuditConstants.SUBMIT_NEW_APP, "UNKNOWN", + "RouterClientRMService", errMsg); + RouterServerUtil.logAndThrowException(errMsg, null); } ApplicationId applicationId = @@ -413,6 +423,10 @@ public SubmitApplicationResponse submitApplication( routerMetrics.incrAppsFailedSubmitted(); String message = "Unable to insert the ApplicationId " + applicationId + " into the FederationStateStore"; + RouterAuditLogger.logFailure(user.toString(), + RouterAuditLogger.AuditConstants.SUBMIT_NEW_APP, "UNKNOWN", + "RouterClientRMService", message, applicationId, + subClusterId); RouterServerUtil.logAndThrowException(message, e); } } else { @@ -430,6 +444,10 @@ public SubmitApplicationResponse submitApplication( + " already submitted on SubCluster " + subClusterId); } else { routerMetrics.incrAppsFailedSubmitted(); + RouterAuditLogger.logFailure(user.toString(), + RouterAuditLogger.AuditConstants.SUBMIT_NEW_APP, "UNKNOWN", + "RouterClientRMService", message, applicationId, + subClusterId); RouterServerUtil.logAndThrowException(message, e); } } @@ -451,6 +469,9 @@ public SubmitApplicationResponse submitApplication( + request.getApplicationSubmissionContext().getApplicationName() + " with appId " + applicationId + " submitted on " + subClusterId); long stopTime = clock.getTime(); + RouterAuditLogger.logSuccess(user.toString(), + RouterAuditLogger.AuditConstants.SUBMIT_NEW_APP, + "RouterClientRMService", applicationId, subClusterId); routerMetrics.succeededAppsSubmitted(stopTime - startTime); return response; } else { @@ -464,7 +485,9 @@ public SubmitApplicationResponse submitApplication( String errMsg = "Application " + request.getApplicationSubmissionContext().getApplicationName() + " with appId " + applicationId + " failed to be submitted."; - LOG.error(errMsg); + RouterAuditLogger.logFailure(user.toString(), + RouterAuditLogger.AuditConstants.SUBMIT_NEW_APP, "UNKNOWN", + "RouterClientRMService", errMsg, applicationId); throw new YarnException(errMsg); } @@ -492,8 +515,11 @@ public KillApplicationResponse forceKillApplication( if (request == null || request.getApplicationId() == null) { routerMetrics.incrAppsFailedKilled(); - RouterServerUtil.logAndThrowException( - "Missing forceKillApplication request or ApplicationId.", null); + String message = "Missing forceKillApplication request or ApplicationId."; + RouterAuditLogger.logFailure(user.toString(), + RouterAuditLogger.AuditConstants.FORCE_KILL_APP, "UNKNOWN", + "RouterClientRMService", message); + RouterServerUtil.logAndThrowException(message, null); } ApplicationId applicationId = request.getApplicationId(); SubClusterId subClusterId = null; @@ -503,6 +529,10 @@ public KillApplicationResponse forceKillApplication( .getApplicationHomeSubCluster(request.getApplicationId()); } catch (YarnException e) { routerMetrics.incrAppsFailedKilled(); + RouterAuditLogger.logFailure(user.toString(), + RouterAuditLogger.AuditConstants.FORCE_KILL_APP, "UNKNOWN", + "RouterClientRMService", "App doesn't exist in FederationStateStore", + applicationId); RouterServerUtil.logAndThrowException("Application " + applicationId + " does not exist in FederationStateStore", e); } @@ -517,9 +547,10 @@ public KillApplicationResponse forceKillApplication( response = clientRMProxy.forceKillApplication(request); } catch (Exception e) { routerMetrics.incrAppsFailedKilled(); - LOG.error("Unable to kill the application report for " - + request.getApplicationId() + "to SubCluster " - + subClusterId.getId(), e); + RouterAuditLogger.logFailure(user.toString(), + RouterAuditLogger.AuditConstants.FORCE_KILL_APP, "UNKNOWN", + "RouterClientRMService", "Unable to kill the application report", + applicationId, subClusterId); throw e; } @@ -530,6 +561,9 @@ public KillApplicationResponse forceKillApplication( long stopTime = clock.getTime(); routerMetrics.succeededAppsKilled(stopTime - startTime); + RouterAuditLogger.logSuccess(user.toString(), + RouterAuditLogger.AuditConstants.FORCE_KILL_APP, + "RouterClientRMService", applicationId); return response; } @@ -557,6 +591,10 @@ public GetApplicationReportResponse getApplicationReport( if (request == null || request.getApplicationId() == null) { routerMetrics.incrAppsFailedRetrieved(); + RouterAuditLogger.logFailure(user.toString(), + RouterAuditLogger.AuditConstants.GET_APP_REPORT, "UNKNOWN", + "RouterClientRMService", + "Missing getApplicationReport request or applicationId information."); RouterServerUtil.logAndThrowException( "Missing getApplicationReport request or applicationId information.", null); @@ -569,6 +607,11 @@ public GetApplicationReportResponse getApplicationReport( .getApplicationHomeSubCluster(request.getApplicationId()); } catch (YarnException e) { routerMetrics.incrAppsFailedRetrieved(); + RouterAuditLogger.logFailure(user.toString(), + RouterAuditLogger.AuditConstants.GET_APP_REPORT, "UNKNOWN", + "RouterClientRMService", + "Application doesn't exist in FederationStateStore", + request.getApplicationId()); RouterServerUtil .logAndThrowException("Application " + request.getApplicationId() + " does not exist in FederationStateStore", e); @@ -582,9 +625,10 @@ public GetApplicationReportResponse getApplicationReport( response = clientRMProxy.getApplicationReport(request); } catch (Exception e) { routerMetrics.incrAppsFailedRetrieved(); - LOG.error("Unable to get the application report for " - + request.getApplicationId() + "to SubCluster " - + subClusterId.getId(), e); + RouterAuditLogger.logFailure(user.toString(), + RouterAuditLogger.AuditConstants.GET_APP_REPORT, "UNKNOWN", + "RouterClientRMService", "unable to get the application report", + request.getApplicationId(), subClusterId); throw e; } @@ -596,6 +640,9 @@ public GetApplicationReportResponse getApplicationReport( long stopTime = clock.getTime(); routerMetrics.succeededAppsRetrieved(stopTime - startTime); + RouterAuditLogger.logSuccess(user.toString(), + RouterAuditLogger.AuditConstants.GET_APP_REPORT, + "RouterClientRMService", request.getApplicationId()); return response; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouterAuditLogger.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouterAuditLogger.java new file mode 100644 index 00000000000..799c209391e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouterAuditLogger.java @@ -0,0 +1,160 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.router; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; +import org.junit.Before; +import org.junit.Test; + +/** + * Tests {@link RouterAuditLogger}. + */ +public class TestRouterAuditLogger { + private static final String USER = "test"; + private static final String OPERATION = "oper"; + private static final String TARGET = "tgt"; + private static final String DESC = "description of an audit log"; + + private static final ApplicationId APPID = mock(ApplicationId.class); + private static final SubClusterId SUBCLUSTERID = mock(SubClusterId.class); + + @Before + public void setUp() throws Exception { + when(APPID.toString()).thenReturn("app_1"); + when(SUBCLUSTERID.toString()).thenReturn("sc0"); + } + + /** + * Test the AuditLog format with key-val pair. + */ + @Test + public void testKeyValLogFormat() throws Exception { + StringBuilder actLog = new StringBuilder(); + StringBuilder expLog = new StringBuilder(); + // add the first k=v pair and check + RouterAuditLogger.start(RouterAuditLogger.Keys.USER, USER, actLog); + expLog.append("USER=test"); + assertEquals(expLog.toString(), actLog.toString()); + + // append another k1=v1 pair to already added k=v and test + RouterAuditLogger.add(RouterAuditLogger.Keys.OPERATION, OPERATION, actLog); + expLog.append("\tOPERATION=oper"); + assertEquals(expLog.toString(), actLog.toString()); + + // append another k1=null pair and test + RouterAuditLogger.add(RouterAuditLogger.Keys.APPID, (String) null, actLog); + expLog.append("\tAPPID=null"); + assertEquals(expLog.toString(), actLog.toString()); + + // now add the target and check of the final string + RouterAuditLogger.add(RouterAuditLogger.Keys.TARGET, TARGET, actLog); + expLog.append("\tTARGET=tgt"); + assertEquals(expLog.toString(), actLog.toString()); + } + + /** + * Test the AuditLog format for successful events. + */ + private void testSuccessLogFormatHelper(ApplicationId appId, + SubClusterId subClusterId) { + // check without the IP + String sLog = RouterAuditLogger + .createSuccessLog(USER, OPERATION, TARGET, appId, subClusterId); + StringBuilder expLog = new StringBuilder(); + expLog.append("USER=test\t"); + expLog.append("OPERATION=oper\tTARGET=tgt\tRESULT=SUCCESS"); + if (appId != null) { + expLog.append("\tAPPID=app_1"); + } + if (subClusterId != null) { + expLog.append("\tSUBCLUSTERID=sc0"); + } + assertEquals(expLog.toString(), sLog); + } + + /** + * Test the AuditLog format for successful events passing nulls. + */ + private void testSuccessLogNulls() { + String sLog = + RouterAuditLogger.createSuccessLog(null, null, null, null, null); + StringBuilder expLog = new StringBuilder(); + expLog.append("USER=null\t"); + expLog.append("OPERATION=null\tTARGET=null\tRESULT=SUCCESS"); + assertEquals(expLog.toString(), sLog); + } + + /** + * Test the AuditLog format for successful events with the various + * parameters. + */ + private void testSuccessLogFormat() { + testSuccessLogFormatHelper(null, null); + testSuccessLogFormatHelper(APPID, null); + testSuccessLogFormatHelper(null, SUBCLUSTERID); + testSuccessLogFormatHelper(APPID, SUBCLUSTERID); + } + + /** + * Test the AuditLog format for failure events. + */ + private void testFailureLogFormatHelper(ApplicationId appId, + SubClusterId subClusterId) { + String fLog = RouterAuditLogger + .createFailureLog(USER, OPERATION, "UNKNOWN", TARGET, DESC, appId, + subClusterId); + StringBuilder expLog = new StringBuilder(); + expLog.append("USER=test\t"); + expLog.append("OPERATION=oper\tTARGET=tgt\tRESULT=FAILURE\t"); + expLog.append("DESCRIPTION=description of an audit log"); + expLog.append("\tPERMISSIONS=UNKNOWN"); + + if (appId != null) { + expLog.append("\tAPPID=app_1"); + } + if (subClusterId != null) { + expLog.append("\tSUBCLUSTERID=sc0"); + } + assertEquals(expLog.toString(), fLog); + } + + /** + * Test the AuditLog format for failure events with the various + * parameters. + */ + private void testFailureLogFormat() { + testFailureLogFormatHelper(null, null); + testFailureLogFormatHelper(APPID, null); + testFailureLogFormatHelper(null, SUBCLUSTERID); + testFailureLogFormatHelper(APPID, SUBCLUSTERID); + } + + /** + * Test {@link RouterAuditLogger}. + */ + @Test + public void testRouterAuditLogger() throws Exception { + testSuccessLogFormat(); + testFailureLogFormat(); + } +}