diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index 6ef69cc..012da11 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -131,6 +131,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppKillEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMoveEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; @@ -638,14 +639,18 @@ public KillApplicationResponse forceKillApplication( } if (application.isAppFinalStateStored()) { - RMAuditLogger.logSuccess(callerUGI.getShortUserName(), - AuditConstants.KILL_APP_REQUEST, "ClientRMService", applicationId); return KillApplicationResponse.newInstance(true); } - this.rmContext.getDispatcher().getEventHandler().handle( - new RMAppEvent(applicationId, RMAppEventType.KILL, - "Application killed by user.")); + String message = "Kill application " + applicationId + " received from " + + callerUGI; + if (null != Server.getRemoteAddress()) { + message += " at " + Server.getRemoteAddress(); + } + + this.rmContext.getDispatcher().getEventHandler() + .handle(new RMAppKillEvent(applicationId, message, + callerUGI, Server.getRemoteAddress())); // For UnmanagedAMs, return true so they don't retry return KillApplicationResponse.newInstance( diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java index db8a46a..a5acd67 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java @@ -88,6 +88,30 @@ static String createSuccessLog(String user, String operation, String target, } /** + * A helper api for creating an audit log for a successful event. + */ + static String createSuccessLog(String user, String operation, String target, + ApplicationId appId, String ip, ApplicationAttemptId attemptId, + ContainerId containerId) { + StringBuilder b = new StringBuilder(); + start(Keys.USER, user, b); + add(Keys.IP, ip, b); + add(Keys.OPERATION, operation, b); + add(Keys.TARGET, target, b); + add(Keys.RESULT, AuditConstants.SUCCESS, b); + if (appId != null) { + add(Keys.APPID, appId.toString(), b); + } + if (attemptId != null) { + add(Keys.APPATTEMPTID, attemptId.toString(), b); + } + if (containerId != null) { + add(Keys.CONTAINERID, containerId.toString(), b); + } + return b.toString(); + } + + /** * Create a readable and parseable audit log string for a successful event. * * @param user User who made the service request to the ResourceManager @@ -152,6 +176,27 @@ public static void logSuccess(String user, String operation, String target, /** * Create a readable and parseable audit log string for a successful event. * + * @param user User who made the service request to the ResourceManager. + * @param operation Operation requested by the user. + * @param target The target on which the operation is being performed. + * @param appId Application Id in which operation was performed. + * @param ip The ip address of the caller. + * + *

+ * Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter + * and hence the value fields should not contains tabs ('\t'). + */ + public static void logSuccess(String user, String operation, String target, + ApplicationId appId, String ipAddr) { + if (LOG.isInfoEnabled()) { + LOG.info( + createSuccessLog(user, operation, target, appId, ipAddr, null, null)); + } + } + + /** + * Create a readable and parseable audit log string for a successful event. + * * @param user User who made the service request. * @param operation Operation requested by the user. * @param target The target on which the operation is being performed. diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index 0ac8331..a77d6e5 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -64,8 +64,10 @@ import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService; import org.apache.hadoop.yarn.server.resourcemanager.RMAppManagerEvent; import org.apache.hadoop.yarn.server.resourcemanager.RMAppManagerEventType; +import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils; +import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState; import org.apache.hadoop.yarn.server.resourcemanager.recovery.Recoverable; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData; @@ -713,7 +715,7 @@ public void handle(RMAppEvent event) { if (oldState != getState()) { LOG.info(appID + " State change from " + oldState + " to " - + getState()); + + getState() + " on event=" + event.getType()); } } finally { this.writeLock.unlock(); @@ -1102,6 +1104,18 @@ public AppKilledTransition() { public void transition(RMAppImpl app, RMAppEvent event) { app.diagnostics.append(event.getDiagnosticMsg()); super.transition(app, event); + if (event instanceof RMAppKillEvent) { + RMAppKillEvent killEvent = (RMAppKillEvent) event; + UserGroupInformation callerUGI = killEvent.getCallerUGI(); + String userName = ""; + if (callerUGI != null) { + userName = callerUGI.getShortUserName(); + } + String remoteIP = killEvent.getRemoteIP(); + RMAuditLogger.logSuccess(userName, AuditConstants.KILL_APP_REQUEST, + "RMAppImpl", event.getApplicationId(), + (remoteIP == null ? "" : remoteIP)); + } }; } @@ -1115,6 +1129,18 @@ public void transition(RMAppImpl app, RMAppEvent event) { app.handler.handle( new RMAppAttemptEvent(app.currentAttempt.getAppAttemptId(), RMAppAttemptEventType.KILL, event.getDiagnosticMsg())); + if (event instanceof RMAppKillEvent) { + RMAppKillEvent killEvent = (RMAppKillEvent) event; + UserGroupInformation callerUGI = killEvent.getCallerUGI(); + String userName = ""; + if (callerUGI != null) { + userName = callerUGI.getShortUserName(); + } + String remoteIP = killEvent.getRemoteIP(); + RMAuditLogger.logSuccess(userName, AuditConstants.KILL_APP_REQUEST, + "RMAppImpl", event.getApplicationId(), + (remoteIP == null ? "" : remoteIP)); + } } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppKillEvent.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppKillEvent.java new file mode 100644 index 0000000..8af8e90 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppKillEvent.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.rmapp; + +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.api.records.ApplicationId; + +public class RMAppKillEvent extends RMAppEvent { + + private final UserGroupInformation callerUGI; + private final String remoteIP; + + public RMAppKillEvent(ApplicationId appId, String diagnostics, + UserGroupInformation callerUGI, String remoteIP) { + super(appId, RMAppEventType.KILL, diagnostics); + this.callerUGI = callerUGI; + this.remoteIP = remoteIP; + } + + public UserGroupInformation getCallerUGI() { + return callerUGI; + } + + public String getRemoteIP() { + return remoteIP; + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java index e61587d..6dd7c5c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler; import java.io.IOException; +import java.net.InetAddress; import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -31,6 +32,8 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.ipc.Server; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -52,8 +55,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; -import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent; -import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppKillEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMoveEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; @@ -544,15 +546,36 @@ public synchronized void killAllAppsInQueue(String queueName) } // generate kill events for each pending/running app for (ApplicationAttemptId app : apps) { - this.rmContext - .getDispatcher() - .getEventHandler() - .handle(new RMAppEvent(app.getApplicationId(), RMAppEventType.KILL, - "Application killed due to expiry of reservation queue " + - queueName + ".")); + String ipAddr = getIpAddressForAuditLogging(); + UserGroupInformation callerUGI = getCallerUGI(); + this.rmContext.getDispatcher().getEventHandler() + .handle(new RMAppKillEvent(app.getApplicationId(), + "Application killed due to expiry of reservation queue " + + queueName + ".", + callerUGI, ipAddr)); } } - + + protected UserGroupInformation getCallerUGI() { + UserGroupInformation callerUGI; + try { + callerUGI = UserGroupInformation.getCurrentUser(); + } catch (IOException e) { + callerUGI = null; + } + return callerUGI; + } + + protected String getIpAddressForAuditLogging() { + InetAddress ip = Server.getRemoteIp(); + // ip address can be null for testcases + String ipAddr = ""; + if (ip != null) { + ipAddr = ip.getHostAddress(); + } + return ipAddr; + } + /** * Process resource update on a node. */ diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index c8ea5a5..22e12ea 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -72,6 +72,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppKillEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType; @@ -717,10 +718,13 @@ private synchronized void addApplicationOnRecovery( //During a restart, this indicates a queue was removed, which is //not presently supported if (!YarnConfiguration.shouldRMFailFast(getConfig())) { + String ipAddr = getIpAddressForAuditLogging(); + UserGroupInformation callerUGI = getCallerUGI(); this.rmContext.getDispatcher().getEventHandler().handle( - new RMAppEvent(applicationId, RMAppEventType.KILL, + new RMAppKillEvent(applicationId, "Application killed on recovery as it was submitted to queue " + - queueName + " which no longer exists after restart.")); + queueName + " which no longer exists after restart.", + callerUGI, ipAddr)); return; } else { String queueErrorMsg = "Queue named " + queueName @@ -736,10 +740,13 @@ private synchronized void addApplicationOnRecovery( // During RM restart, this means leaf queue was converted to a parent // queue, which is not supported for running apps. if (!YarnConfiguration.shouldRMFailFast(getConfig())) { + String ipAddr = getIpAddressForAuditLogging(); + UserGroupInformation callerUGI = getCallerUGI(); this.rmContext.getDispatcher().getEventHandler().handle( - new RMAppEvent(applicationId, RMAppEventType.KILL, + new RMAppKillEvent(applicationId, "Application killed on recovery as it was submitted to queue " + - queueName + " which is no longer a leaf queue after restart.")); + queueName + " which is no longer a leaf queue after restart.", + callerUGI, ipAddr)); return; } else { String queueErrorMsg = "Queue named " + queueName diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java index b5ad74a..1c540de 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java @@ -547,11 +547,16 @@ public void testAppRecoverPath() throws IOException { public void testAppNewKill() throws IOException { LOG.info("--- START: testAppNewKill ---"); + UserGroupInformation fooUser = UserGroupInformation.createUserForTesting( + "fooTestAppNewKill", new String[] { "foo_group" }); + UserGroupInformation testUGI = UserGroupInformation.createUserForTesting( + fooUser.getShortUserName(), fooUser.getGroupNames()); + RMApp application = createNewTestApp(null); // NEW => KILLED event RMAppEventType.KILL - RMAppEvent event = - new RMAppEvent(application.getApplicationId(), RMAppEventType.KILL, - "Application killed by user."); + RMAppEvent event = new RMAppKillEvent(application.getApplicationId(), + "Application killed by user.", testUGI, + "127.0.0.1"); application.handle(event); rmDispatcher.await(); sendAppUpdateSavedEvent(application); @@ -602,9 +607,15 @@ public void testAppNewSavingKill() throws IOException { RMApp application = testCreateAppNewSaving(null); // NEW_SAVING => KILLED event RMAppEventType.KILL - RMAppEvent event = - new RMAppEvent(application.getApplicationId(), RMAppEventType.KILL, - "Application killed by user."); + UserGroupInformation fooUser = UserGroupInformation.createUserForTesting( + "fooTestAppNewSavingKill", new String[] { "foo_group" }); + UserGroupInformation testUGI = UserGroupInformation.createUserForTesting( + fooUser.getShortUserName(), fooUser.getGroupNames()); + + RMAppEvent event = new RMAppKillEvent(application.getApplicationId(), + "Application killed by user.", testUGI, + "127.0.0.1"); + application.handle(event); rmDispatcher.await(); sendAppUpdateSavedEvent(application); @@ -651,10 +662,17 @@ public void testAppSubmittedRejected() throws IOException { public void testAppSubmittedKill() throws IOException, InterruptedException { LOG.info("--- START: testAppSubmittedKill---"); RMApp application = testCreateAppSubmittedNoRecovery(null); + + UserGroupInformation fooUser = UserGroupInformation.createUserForTesting( + "fooTestAppSubmittedKill", new String[] { "foo_group" }); + UserGroupInformation testUGI = UserGroupInformation.createUserForTesting( + fooUser.getShortUserName(), fooUser.getGroupNames()); + // SUBMITTED => KILLED event RMAppEventType.KILL - RMAppEvent event = - new RMAppEvent(application.getApplicationId(), RMAppEventType.KILL, - "Application killed by user."); + RMAppEvent event = new RMAppKillEvent(application.getApplicationId(), + "Application killed by user.", testUGI, + "127.0.0.1"); + application.handle(event); rmDispatcher.await(); sendAppUpdateSavedEvent(application); @@ -704,9 +722,15 @@ public void testAppAcceptedKill() throws IOException, InterruptedException { LOG.info("--- START: testAppAcceptedKill ---"); RMApp application = testCreateAppAccepted(null); // ACCEPTED => KILLED event RMAppEventType.KILL - RMAppEvent event = - new RMAppEvent(application.getApplicationId(), RMAppEventType.KILL, - "Application killed by user."); + UserGroupInformation fooUser = UserGroupInformation.createUserForTesting( + "fooTestAppAcceptedKill", new String[] { "foo_group" }); + UserGroupInformation testUGI = UserGroupInformation.createUserForTesting( + fooUser.getShortUserName(), fooUser.getGroupNames()); + + RMAppEvent event = new RMAppKillEvent(application.getApplicationId(), + "Application killed by user.", testUGI, + "127.0.0.1"); + application.handle(event); rmDispatcher.await(); @@ -752,9 +776,16 @@ public void testAppRunningKill() throws IOException { RMApp application = testCreateAppRunning(null); // RUNNING => KILLED event RMAppEventType.KILL - RMAppEvent event = - new RMAppEvent(application.getApplicationId(), RMAppEventType.KILL, - "Application killed by user."); + UserGroupInformation fooUser = UserGroupInformation.createUserForTesting( + "fooTestAppRunningKill", new String[] { "foo_group" }); + UserGroupInformation testUGI = UserGroupInformation.createUserForTesting( + fooUser.getShortUserName(), fooUser.getGroupNames()); + + // SUBMITTED => KILLED event RMAppEventType.KILL + RMAppEvent event = new RMAppKillEvent(application.getApplicationId(), + "Application killed by user.", testUGI, + "127.0.0.1"); + application.handle(event); rmDispatcher.await(); @@ -918,9 +949,16 @@ public void testAppKilledKilled() throws IOException { RMApp application = testCreateAppRunning(null); // RUNNING => KILLED event RMAppEventType.KILL - RMAppEvent event = - new RMAppEvent(application.getApplicationId(), RMAppEventType.KILL, - "Application killed by user."); + UserGroupInformation fooUser = UserGroupInformation.createUserForTesting( + "fooTestAppRunningKill", new String[] { "foo_group" }); + UserGroupInformation testUGI = UserGroupInformation.createUserForTesting( + fooUser.getShortUserName(), fooUser.getGroupNames()); + + // SUBMITTED => KILLED event RMAppEventType.KILL + RMAppEvent event = new RMAppKillEvent(application.getApplicationId(), + "Application killed by user.", testUGI, + "127.0.0.1"); + application.handle(event); rmDispatcher.await(); sendAttemptUpdateSavedEvent(application);