diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java index 803dc01..a58d8ac 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java @@ -36,6 +36,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerReport; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptHistoryData; import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationHistoryData; import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerHistoryData; @@ -90,7 +91,7 @@ protected ApplicationHistoryStore createApplicationHistoryStore( @Override public ContainerReport getAMContainer(ApplicationAttemptId appAttemptId) - throws IOException { + throws YarnException, IOException { ApplicationReport app = getApplication(appAttemptId.getApplicationId()); return convertToContainerReport(historyStore.getAMContainer(appAttemptId), @@ -99,7 +100,7 @@ public ContainerReport getAMContainer(ApplicationAttemptId appAttemptId) @Override public Map getAllApplications() - throws IOException { + throws YarnException, IOException { Map histData = historyStore.getAllApplications(); HashMap applicationsReport = @@ -114,7 +115,7 @@ public ContainerReport getAMContainer(ApplicationAttemptId appAttemptId) @Override public ApplicationReport getApplication(ApplicationId appId) - throws IOException { + throws YarnException, IOException { return convertToApplicationReport(historyStore.getApplication(appId)); } @@ -171,14 +172,14 @@ private ApplicationAttemptReport convertToApplicationAttemptReport( @Override public ApplicationAttemptReport getApplicationAttempt( - ApplicationAttemptId appAttemptId) throws IOException { + ApplicationAttemptId appAttemptId) throws YarnException, IOException { return convertToApplicationAttemptReport(historyStore .getApplicationAttempt(appAttemptId)); } @Override public Map - getApplicationAttempts(ApplicationId appId) throws IOException { + getApplicationAttempts(ApplicationId appId) throws YarnException, IOException { Map histData = historyStore.getApplicationAttempts(appId); HashMap applicationAttemptsReport = @@ -193,7 +194,7 @@ public ApplicationAttemptReport getApplicationAttempt( @Override public ContainerReport getContainer(ContainerId containerId) - throws IOException { + throws YarnException, IOException { ApplicationReport app = getApplication(containerId.getApplicationAttemptId().getApplicationId()); return convertToContainerReport(historyStore.getContainer(containerId), @@ -220,7 +221,7 @@ private ContainerReport convertToContainerReport( @Override public Map getContainers( - ApplicationAttemptId appAttemptId) throws IOException { + ApplicationAttemptId appAttemptId) throws YarnException, IOException { ApplicationReport app = getApplication(appAttemptId.getApplicationId()); Map histData = diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java index 5b498f7..89e8485 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.applicationhistoryservice; import java.io.IOException; +import java.util.Arrays; import java.util.EnumSet; import java.util.HashMap; import java.util.List; @@ -27,6 +28,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -48,6 +50,7 @@ import org.apache.hadoop.yarn.server.metrics.AppAttemptMetricsConstants; import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants; import org.apache.hadoop.yarn.server.metrics.ContainerMetricsConstants; +import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.server.timeline.NameValuePair; import org.apache.hadoop.yarn.server.timeline.TimelineDataManager; import org.apache.hadoop.yarn.server.timeline.TimelineReader.Field; @@ -59,12 +62,15 @@ ApplicationHistoryManager { private TimelineDataManager timelineDataManager; + private ApplicationACLsManager aclsManager; private String serverHttpAddress; public ApplicationHistoryManagerOnTimelineStore( - TimelineDataManager timelineDataManager) { + TimelineDataManager timelineDataManager, + ApplicationACLsManager aclsManager) { super(ApplicationHistoryManagerOnTimelineStore.class.getName()); this.timelineDataManager = timelineDataManager; + this.aclsManager = aclsManager; } @Override @@ -76,34 +82,34 @@ protected void serviceInit(Configuration conf) throws Exception { @Override public ApplicationReport getApplication(ApplicationId appId) - throws IOException { + throws YarnException, IOException { TimelineEntity entity = null; try { entity = timelineDataManager.getEntity( ApplicationMetricsConstants.ENTITY_TYPE, appId.toString(), EnumSet.allOf(Field.class), - UserGroupInformation.getCurrentUser()); + UserGroupInformation.getLoginUser()); } catch (YarnException e) { throw new IOException(e); } if (entity == null) { return null; } else { - return generateApplicationReport(entity); + return generateApplicationReport(entity).appReport; } } @Override public Map getAllApplications() - throws IOException { + throws YarnException, IOException { TimelineEntities entities = null; try { entities = timelineDataManager.getEntities( ApplicationMetricsConstants.ENTITY_TYPE, null, null, null, null, null, null, Long.MAX_VALUE, EnumSet.allOf(Field.class), - UserGroupInformation.getCurrentUser()); + UserGroupInformation.getLoginUser()); } catch (YarnException e) { throw new IOException(e); } @@ -111,8 +117,8 @@ public ApplicationReport getApplication(ApplicationId appId) new HashMap(); if (entities != null && entities.getEntities() != null) { for (TimelineEntity entity : entities.getEntities()) { - ApplicationReport app = generateApplicationReport(entity); - apps.put(app.getApplicationId(), app); + ApplicationReportExt app = generateApplicationReport(entity); + apps.put(app.appReport.getApplicationId(), app.appReport); } } return apps; @@ -121,7 +127,8 @@ public ApplicationReport getApplication(ApplicationId appId) @Override public Map getApplicationAttempts( - ApplicationId appId) throws IOException { + ApplicationId appId) throws YarnException, IOException { + checkAccess(appId); TimelineEntities entities = null; try { entities = @@ -131,7 +138,7 @@ public ApplicationReport getApplication(ApplicationId appId) AppAttemptMetricsConstants.PARENT_PRIMARY_FILTER, appId .toString()), null, null, null, null, null, Long.MAX_VALUE, EnumSet.allOf(Field.class), - UserGroupInformation.getCurrentUser()); + UserGroupInformation.getLoginUser()); } catch (YarnException e) { throw new IOException(e); } @@ -149,14 +156,15 @@ public ApplicationReport getApplication(ApplicationId appId) @Override public ApplicationAttemptReport getApplicationAttempt( - ApplicationAttemptId appAttemptId) throws IOException { + ApplicationAttemptId appAttemptId) throws YarnException, IOException { + checkAccess(appAttemptId.getApplicationId()); TimelineEntity entity = null; try { entity = timelineDataManager.getEntity( AppAttemptMetricsConstants.ENTITY_TYPE, appAttemptId.toString(), EnumSet.allOf(Field.class), - UserGroupInformation.getCurrentUser()); + UserGroupInformation.getLoginUser()); } catch (YarnException e) { throw new IOException(e); } @@ -169,7 +177,8 @@ public ApplicationAttemptReport getApplicationAttempt( @Override public ContainerReport getContainer(ContainerId containerId) - throws IOException { + throws YarnException, IOException { + checkAccess(containerId.getApplicationAttemptId().getApplicationId()); ApplicationReport app = getApplication(containerId.getApplicationAttemptId().getApplicationId()); if (app == null) { @@ -181,7 +190,7 @@ public ContainerReport getContainer(ContainerId containerId) timelineDataManager.getEntity( ContainerMetricsConstants.ENTITY_TYPE, containerId.toString(), EnumSet.allOf(Field.class), - UserGroupInformation.getCurrentUser()); + UserGroupInformation.getLoginUser()); } catch (YarnException e) { throw new IOException(e); } @@ -194,7 +203,8 @@ public ContainerReport getContainer(ContainerId containerId) @Override public ContainerReport getAMContainer(ApplicationAttemptId appAttemptId) - throws IOException { + throws YarnException, IOException { + checkAccess(appAttemptId.getApplicationId()); ApplicationAttemptReport appAttempt = getApplicationAttempt(appAttemptId); if (appAttempt == null) { return null; @@ -205,7 +215,8 @@ public ContainerReport getAMContainer(ApplicationAttemptId appAttemptId) @Override public Map getContainers( - ApplicationAttemptId appAttemptId) throws IOException { + ApplicationAttemptId appAttemptId) throws YarnException, IOException { + checkAccess(appAttemptId.getApplicationId()); ApplicationReport app = getApplication(appAttemptId.getApplicationId()); if (app == null) { @@ -220,7 +231,7 @@ public ContainerReport getAMContainer(ApplicationAttemptId appAttemptId) ContainerMetricsConstants.PARENT_PRIMARIY_FILTER, appAttemptId.toString()), null, null, null, null, null, Long.MAX_VALUE, EnumSet.allOf(Field.class), - UserGroupInformation.getCurrentUser()); + UserGroupInformation.getLoginUser()); } catch (YarnException e) { throw new IOException(e); } @@ -236,8 +247,8 @@ public ContainerReport getAMContainer(ApplicationAttemptId appAttemptId) return containers; } - private static ApplicationReport convertToApplicationReport( - TimelineEntity entity) { + private static ApplicationReportExt convertToApplicationReport( + TimelineEntity entity, boolean ACLsOnly) { String user = null; String queue = null; String name = null; @@ -248,6 +259,8 @@ private static ApplicationReport convertToApplicationReport( String diagnosticsInfo = null; FinalApplicationStatus finalStatus = FinalApplicationStatus.UNDEFINED; YarnApplicationState state = null; + Map appViewACLs = + new HashMap(); Map entityInfo = entity.getOtherInfo(); if (entityInfo != null) { if (entityInfo.containsKey(ApplicationMetricsConstants.USER_ENTITY_INFO)) { @@ -255,6 +268,16 @@ private static ApplicationReport convertToApplicationReport( entityInfo.get(ApplicationMetricsConstants.USER_ENTITY_INFO) .toString(); } + if (entityInfo.containsKey(ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO)) { + for (String appViewACLsEach : Arrays.asList(entityInfo.get( + ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO).toString().split(":"))) { + appViewACLs.put(ApplicationAccessType.VIEW_APP, appViewACLsEach); + } + } + // Return early if we just want the ACLs information + if (ACLsOnly) { + return new ApplicationReportExt(null, appViewACLs, user); + } if (entityInfo.containsKey(ApplicationMetricsConstants.QUEUE_ENTITY_INFO)) { queue = entityInfo.get(ApplicationMetricsConstants.QUEUE_ENTITY_INFO) @@ -317,7 +340,7 @@ private static ApplicationReport convertToApplicationReport( } } } - return ApplicationReport.newInstance( + return new ApplicationReportExt(ApplicationReport.newInstance( ConverterUtils.toApplicationId(entity.getEntityId()), latestApplicationAttemptId, user, queue, name, null, @@ -333,7 +356,7 @@ private static ApplicationReport convertToApplicationReport( null, 1.0F, type, - null); + null), appViewACLs, user); } private static ApplicationAttemptReport convertToApplicationAttemptReport( @@ -508,19 +531,86 @@ private static ContainerReport convertToContainerReport( createdTime, finishedTime, diagnosticsInfo, logUrl, exitStatus, state); } - private ApplicationReport generateApplicationReport(TimelineEntity entity) - throws IOException { - ApplicationReport app = convertToApplicationReport(entity); - if (app != null && app.getCurrentApplicationAttemptId() != null) { - ApplicationAttemptReport appAttempt = - getApplicationAttempt(app.getCurrentApplicationAttemptId()); - if (appAttempt != null) { - app.setHost(appAttempt.getHost()); - app.setRpcPort(appAttempt.getRpcPort()); - app.setTrackingUrl(appAttempt.getTrackingUrl()); - app.setOriginalTrackingUrl(appAttempt.getOriginalTrackingUrl()); + private ApplicationReportExt generateApplicationReport(TimelineEntity entity) + throws YarnException, IOException { + ApplicationReportExt app = convertToApplicationReport(entity, false); + aclsManager.addApplication(app.appReport.getApplicationId(), app.appViewACLs); + try { + if (aclsManager.checkAccess(UserGroupInformation.getCurrentUser(), + ApplicationAccessType.VIEW_APP, app.appOwner, app.appReport.getApplicationId())) { + if (app != null && app.appReport.getCurrentApplicationAttemptId() != null) { + ApplicationAttemptReport appAttempt = + getApplicationAttempt(app.appReport.getCurrentApplicationAttemptId()); + if (appAttempt != null) { + app.appReport.setHost(appAttempt.getHost()); + app.appReport.setRpcPort(appAttempt.getRpcPort()); + app.appReport.setTrackingUrl(appAttempt.getTrackingUrl()); + app.appReport.setOriginalTrackingUrl(appAttempt.getOriginalTrackingUrl()); + } + } + } else { + app.appReport.setDiagnostics(null); + app.appReport.setCurrentApplicationAttemptId(null); } + } finally { + aclsManager.removeApplication(app.appReport.getApplicationId()); + } + if (app.appReport.getCurrentApplicationAttemptId() == null) { + app.appReport.setCurrentApplicationAttemptId( + ApplicationAttemptId.newInstance(app.appReport.getApplicationId(), -1)); } return app; } + + private ApplicationReportExt getAppViewACLs(ApplicationId appId) + throws IOException { + TimelineEntity entity = null; + try { + entity = + timelineDataManager.getEntity( + ApplicationMetricsConstants.ENTITY_TYPE, + appId.toString(), EnumSet.allOf(Field.class), + UserGroupInformation.getLoginUser()); + } catch (YarnException e) { + throw new IOException(e); + } + if (entity == null) { + return null; + } else { + return convertToApplicationReport(entity, true); + } + } + + private void checkAccess(ApplicationId appId) + throws YarnException, IOException { + ApplicationReportExt app = getAppViewACLs(appId); + if (app.appViewACLs != null) { + aclsManager.addApplication(appId, app.appViewACLs); + try { + if (!aclsManager.checkAccess(UserGroupInformation.getCurrentUser(), + ApplicationAccessType.VIEW_APP, app.appOwner, appId)) { + throw new YarnException("User " + + UserGroupInformation.getCurrentUser().getShortUserName() + + " does not have privilage to see this application " + appId); + } + } finally { + aclsManager.removeApplication(appId); + } + } + } + + private static class ApplicationReportExt { + private ApplicationReport appReport; + private Map appViewACLs; + private String appOwner; + + public ApplicationReportExt( + ApplicationReport appReport, + Map appViewACLs, + String appOwner) { + this.appReport = appReport; + this.appViewACLs = appViewACLs; + this.appOwner = appOwner; + } + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java index 1a0e452..eec7014 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java @@ -38,6 +38,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp; +import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore; import org.apache.hadoop.yarn.server.timeline.TimelineDataManager; import org.apache.hadoop.yarn.server.timeline.TimelineStore; @@ -61,6 +62,7 @@ .getLog(ApplicationHistoryServer.class); protected ApplicationHistoryClientService ahsClientService; + protected ApplicationACLsManager aclsManager; protected ApplicationHistoryManager historyManager; protected TimelineStore timelineStore; protected TimelineDelegationTokenSecretManagerService secretManagerService; @@ -83,6 +85,7 @@ protected void serviceInit(Configuration conf) throws Exception { timelineDataManager = createTimelineDataManager(conf); // init generic history service afterwards + aclsManager = createApplicationACLsManager(conf); historyManager = createApplicationHistoryManager(conf); ahsClientService = createApplicationHistoryClientService(historyManager); addService(ahsClientService); @@ -152,10 +155,16 @@ public static void main(String[] args) { launchAppHistoryServer(args); } + protected ApplicationACLsManager createApplicationACLsManager( + Configuration conf) { + return new ApplicationACLsManager(conf); + } + protected ApplicationHistoryManager createApplicationHistoryManager( Configuration conf) { if (conf.get(YarnConfiguration.APPLICATION_HISTORY_STORE) == null) { - return new ApplicationHistoryManagerOnTimelineStore(timelineDataManager); + return new ApplicationHistoryManagerOnTimelineStore( + timelineDataManager, aclsManager); } else { return new ApplicationHistoryManagerImpl(); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java index 65eafc6..88b93d4 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java @@ -18,10 +18,16 @@ package org.apache.hadoop.yarn.server.applicationhistoryservice; +import java.lang.reflect.UndeclaredThrowableException; +import java.security.PrivilegedExceptionAction; +import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.Map; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.SaslRpcServer.AuthMethod; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -42,40 +48,75 @@ import org.apache.hadoop.yarn.server.metrics.AppAttemptMetricsConstants; import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants; import org.apache.hadoop.yarn.server.metrics.ContainerMetricsConstants; +import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore; import org.apache.hadoop.yarn.server.timeline.TimelineDataManager; import org.apache.hadoop.yarn.server.timeline.TimelineStore; import org.apache.hadoop.yarn.server.timeline.security.TimelineACLsManager; -import org.junit.AfterClass; +import org.junit.After; import org.junit.Assert; +import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; +@RunWith(Parameterized.class) public class TestApplicationHistoryManagerOnTimelineStore { - private static ApplicationHistoryManagerOnTimelineStore historyManager; private static final int SCALE = 5; + private static TimelineStore store; + + private ApplicationHistoryManagerOnTimelineStore historyManager; + private UserGroupInformation callerUGI; + private Configuration conf; @BeforeClass - public static void setup() throws Exception { - YarnConfiguration conf = new YarnConfiguration(); - TimelineStore store = new MemoryTimelineStore(); + public static void prepareStore() throws Exception { + store = new MemoryTimelineStore(); prepareTimelineStore(store); - TimelineACLsManager aclsManager = new TimelineACLsManager(conf); + } + + @Before + public void setup() throws Exception { + // Only test the ACLs of the generic history + TimelineACLsManager aclsManager = new TimelineACLsManager(new YarnConfiguration()); TimelineDataManager dataManager = new TimelineDataManager(store, aclsManager); - historyManager = new ApplicationHistoryManagerOnTimelineStore(dataManager); + ApplicationACLsManager appAclsManager = new ApplicationACLsManager(conf); + historyManager = + new ApplicationHistoryManagerOnTimelineStore(dataManager, appAclsManager); historyManager.init(conf); historyManager.start(); } - @AfterClass - public static void tearDown() { + @After + public void tearDown() { if (historyManager != null) { historyManager.stop(); } } + @Parameters + public static Collection callers() { + // user1 is the owner + // user2 is the authorized user + // user3 is the unauthorized user + // admin is the admin acl + return Arrays.asList( + new Object[][] { { "" }, { "user1" }, { "user2" }, { "user3" }, { "admin" } }); + } + + public TestApplicationHistoryManagerOnTimelineStore(String caller) { + conf = new YarnConfiguration(); + if (!caller.equals("")) { + callerUGI = UserGroupInformation.createRemoteUser(caller, AuthMethod.SIMPLE); + conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); + conf.set(YarnConfiguration.YARN_ADMIN_ACL, "admin"); + } + } + private static void prepareTimelineStore(TimelineStore store) throws Exception { for (int i = 1; i <= SCALE; ++i) { @@ -101,23 +142,46 @@ private static void prepareTimelineStore(TimelineStore store) @Test public void testGetApplicationReport() throws Exception { - ApplicationId appId = ApplicationId.newInstance(0, 1); - ApplicationReport app = historyManager.getApplication(appId); + final ApplicationId appId = ApplicationId.newInstance(0, 1); + ApplicationReport app; + if (callerUGI == null) { + app = historyManager.getApplication(appId); + } else { + app = + callerUGI.doAs(new PrivilegedExceptionAction () { + @Override + public ApplicationReport run() throws Exception { + return historyManager.getApplication(appId); + } + }); + } Assert.assertNotNull(app); Assert.assertEquals(appId, app.getApplicationId()); Assert.assertEquals("test app", app.getName()); Assert.assertEquals("test app type", app.getApplicationType()); - Assert.assertEquals("test user", app.getUser()); + Assert.assertEquals("user1", app.getUser()); Assert.assertEquals("test queue", app.getQueue()); Assert.assertEquals(Integer.MAX_VALUE + 2L, app.getStartTime()); Assert.assertEquals(Integer.MAX_VALUE + 3L, app.getFinishTime()); Assert.assertTrue(Math.abs(app.getProgress() - 1.0F) < 0.0001); - Assert.assertEquals("test host", app.getHost()); - Assert.assertEquals(-100, app.getRpcPort()); - Assert.assertEquals("test tracking url", app.getTrackingUrl()); - Assert.assertEquals("test original tracking url", - app.getOriginalTrackingUrl()); - Assert.assertEquals("test diagnostics info", app.getDiagnostics()); + if (callerUGI != null && callerUGI.getShortUserName().equals("user3")) { + Assert.assertEquals(ApplicationAttemptId.newInstance(appId, -1), + app.getCurrentApplicationAttemptId()); + Assert.assertEquals(null, app.getHost()); + Assert.assertEquals(-1, app.getRpcPort()); + Assert.assertEquals(null, app.getTrackingUrl()); + Assert.assertEquals(null, app.getOriginalTrackingUrl()); + Assert.assertEquals(null, app.getDiagnostics()); + } else { + Assert.assertEquals(ApplicationAttemptId.newInstance(appId, 1), + app.getCurrentApplicationAttemptId()); + Assert.assertEquals("test host", app.getHost()); + Assert.assertEquals(-100, app.getRpcPort()); + Assert.assertEquals("test tracking url", app.getTrackingUrl()); + Assert.assertEquals("test original tracking url", + app.getOriginalTrackingUrl()); + Assert.assertEquals("test diagnostics info", app.getDiagnostics()); + } Assert.assertEquals(FinalApplicationStatus.UNDEFINED, app.getFinalApplicationStatus()); Assert.assertEquals(YarnApplicationState.FINISHED, @@ -126,10 +190,31 @@ public void testGetApplicationReport() throws Exception { @Test public void testGetApplicationAttemptReport() throws Exception { - ApplicationAttemptId appAttemptId = + final ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1); - ApplicationAttemptReport appAttempt = - historyManager.getApplicationAttempt(appAttemptId); + ApplicationAttemptReport appAttempt; + if (callerUGI == null) { + appAttempt = historyManager.getApplicationAttempt(appAttemptId); + } else { + try { + appAttempt = + callerUGI.doAs(new PrivilegedExceptionAction () { + @Override + public ApplicationAttemptReport run() throws Exception { + return historyManager.getApplicationAttempt(appAttemptId); + } + }); + } catch (UndeclaredThrowableException e) { + if (callerUGI != null && callerUGI.getShortUserName().equals("user3")) { + if (e.getCause().getMessage().contains( + "does not have privilage to see this application")) { + // The exception is expected + return; + } + } + throw e; + } + } Assert.assertNotNull(appAttempt); Assert.assertEquals(appAttemptId, appAttempt.getApplicationAttemptId()); Assert.assertEquals(ContainerId.newInstance(appAttemptId, 1), @@ -146,10 +231,32 @@ public void testGetApplicationAttemptReport() throws Exception { @Test public void testGetContainerReport() throws Exception { - ContainerId containerId = + final ContainerId containerId = ContainerId.newInstance(ApplicationAttemptId.newInstance( ApplicationId.newInstance(0, 1), 1), 1); - ContainerReport container = historyManager.getContainer(containerId); + ContainerReport container; + if (callerUGI == null) { + container = historyManager.getContainer(containerId); + } else { + try { + container = + callerUGI.doAs(new PrivilegedExceptionAction () { + @Override + public ContainerReport run() throws Exception { + return historyManager.getContainer(containerId); + } + }); + } catch (UndeclaredThrowableException e) { + if (callerUGI != null && callerUGI.getShortUserName().equals("user3")) { + if (e.getCause().getMessage().contains( + "does not have privilage to see this application")) { + // The exception is expected + return; + } + } + throw e; + } + } Assert.assertNotNull(container); Assert.assertEquals(Integer.MAX_VALUE + 1L, container.getCreationTime()); Assert.assertEquals(Integer.MAX_VALUE + 2L, container.getFinishTime()); @@ -164,7 +271,7 @@ public void testGetContainerReport() throws Exception { Assert.assertEquals(-1, container.getContainerExitStatus()); Assert.assertEquals("http://0.0.0.0:8188/applicationhistory/logs/" + "test host:-100/container_0_0001_01_000001/" - + "container_0_0001_01_000001/test user", container.getLogUrl()); + + "container_0_0001_01_000001/user1", container.getLogUrl()); } @Test @@ -177,29 +284,92 @@ public void testGetApplications() throws Exception { @Test public void testGetApplicationAttempts() throws Exception { - Collection appAttempts = - historyManager.getApplicationAttempts(ApplicationId.newInstance(0, 1)) - .values(); + final ApplicationId appId = ApplicationId.newInstance(0, 1); + Collection appAttempts; + if (callerUGI == null) { + appAttempts = historyManager.getApplicationAttempts(appId).values(); + } else { + try { + appAttempts = callerUGI.doAs( + new PrivilegedExceptionAction> () { + @Override + public Collection run() throws Exception { + return historyManager.getApplicationAttempts(appId).values(); + } + }); + } catch (UndeclaredThrowableException e) { + if (callerUGI != null && callerUGI.getShortUserName().equals("user3")) { + if (e.getCause().getMessage().contains( + "does not have privilage to see this application")) { + // The exception is expected + return; + } + } + throw e; + } + } Assert.assertNotNull(appAttempts); Assert.assertEquals(SCALE, appAttempts.size()); } @Test public void testGetContainers() throws Exception { - Collection containers = - historyManager - .getContainers( - ApplicationAttemptId.newInstance( - ApplicationId.newInstance(0, 1), 1)).values(); + final ApplicationAttemptId appAttemptId = + ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1); + Collection containers; + if (callerUGI == null) { + containers = historyManager.getContainers(appAttemptId).values(); + } else { + try { + containers = callerUGI.doAs( + new PrivilegedExceptionAction> () { + @Override + public Collection run() throws Exception { + return historyManager.getContainers(appAttemptId).values(); + } + }); + } catch (UndeclaredThrowableException e) { + if (callerUGI != null && callerUGI.getShortUserName().equals("user3")) { + if (e.getCause().getMessage().contains( + "does not have privilage to see this application")) { + // The exception is expected + return; + } + } + throw e; + } + } Assert.assertNotNull(containers); Assert.assertEquals(SCALE, containers.size()); } @Test public void testGetAMContainer() throws Exception { - ApplicationAttemptId appAttemptId = + final ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1); - ContainerReport container = historyManager.getAMContainer(appAttemptId); + ContainerReport container; + if (callerUGI == null) { + container = historyManager.getAMContainer(appAttemptId); + } else { + try { + container = + callerUGI.doAs(new PrivilegedExceptionAction () { + @Override + public ContainerReport run() throws Exception { + return historyManager.getAMContainer(appAttemptId); + } + }); + } catch (UndeclaredThrowableException e) { + if (callerUGI != null && callerUGI.getShortUserName().equals("user3")) { + if (e.getCause().getMessage().contains( + "does not have privilage to see this application")) { + // The exception is expected + return; + } + } + throw e; + } + } Assert.assertNotNull(container); Assert.assertEquals(appAttemptId, container.getContainerId() .getApplicationAttemptId()); @@ -210,14 +380,18 @@ private static TimelineEntity createApplicationTimelineEntity( TimelineEntity entity = new TimelineEntity(); entity.setEntityType(ApplicationMetricsConstants.ENTITY_TYPE); entity.setEntityId(appId.toString()); + entity.addPrimaryFilter( + TimelineStore.SystemFilter.ENTITY_OWNER.toString(), "yarn"); Map entityInfo = new HashMap(); entityInfo.put(ApplicationMetricsConstants.NAME_ENTITY_INFO, "test app"); entityInfo.put(ApplicationMetricsConstants.TYPE_ENTITY_INFO, "test app type"); - entityInfo.put(ApplicationMetricsConstants.USER_ENTITY_INFO, "test user"); + entityInfo.put(ApplicationMetricsConstants.USER_ENTITY_INFO, "user1"); entityInfo.put(ApplicationMetricsConstants.QUEUE_ENTITY_INFO, "test queue"); entityInfo.put(ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO, Integer.MAX_VALUE + 1L); + entityInfo.put(ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO, + "user2"); entity.setOtherInfo(entityInfo); TimelineEvent tEvent = new TimelineEvent(); tEvent.setEventType(ApplicationMetricsConstants.CREATED_EVENT_TYPE); @@ -248,6 +422,8 @@ private static TimelineEntity createAppAttemptTimelineEntity( entity.setEntityId(appAttemptId.toString()); entity.addPrimaryFilter(AppAttemptMetricsConstants.PARENT_PRIMARY_FILTER, appAttemptId.getApplicationId().toString()); + entity.addPrimaryFilter( + TimelineStore.SystemFilter.ENTITY_OWNER.toString(), "yarn"); TimelineEvent tEvent = new TimelineEvent(); tEvent.setEventType(AppAttemptMetricsConstants.REGISTERED_EVENT_TYPE); tEvent.setTimestamp(Integer.MAX_VALUE + 1L); @@ -287,6 +463,8 @@ private static TimelineEntity createContainerEntity(ContainerId containerId) { entity.setEntityId(containerId.toString()); entity.addPrimaryFilter(ContainerMetricsConstants.PARENT_PRIMARIY_FILTER, containerId.getApplicationAttemptId().toString()); + entity.addPrimaryFilter( + TimelineStore.SystemFilter.ENTITY_OWNER.toString(), "yarn"); Map entityInfo = new HashMap(); entityInfo.put(ContainerMetricsConstants.ALLOCATED_MEMORY_ENTITY_INFO, -1); entityInfo.put(ContainerMetricsConstants.ALLOCATED_VCORE_ENTITY_INFO, -1); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ApplicationContext.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ApplicationContext.java index 78ae0dd..eaf7a13 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ApplicationContext.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ApplicationContext.java @@ -29,6 +29,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerReport; +import org.apache.hadoop.yarn.exceptions.YarnException; @Public @Unstable @@ -40,21 +41,25 @@ * @param appId * * @return {@link ApplicationReport} for the ApplicationId. + * @throws YarnException * @throws IOException */ @Public @Unstable - ApplicationReport getApplication(ApplicationId appId) throws IOException; + ApplicationReport getApplication(ApplicationId appId) + throws YarnException, IOException; /** * This method returns all Application {@link ApplicationReport}s * * @return map of {@link ApplicationId} to {@link ApplicationReport}s. + * @throws YarnException * @throws IOException */ @Public @Unstable - Map getAllApplications() throws IOException; + Map getAllApplications() + throws YarnException, IOException; /** * Application can have multiple application attempts @@ -64,12 +69,13 @@ * @param appId * * @return all {@link ApplicationAttemptReport}s for the Application. + * @throws YarnException * @throws IOException */ @Public @Unstable Map getApplicationAttempts( - ApplicationId appId) throws IOException; + ApplicationId appId) throws YarnException, IOException; /** * This method returns {@link ApplicationAttemptReport} for specified @@ -78,12 +84,13 @@ * @param appAttemptId * {@link ApplicationAttemptId} * @return {@link ApplicationAttemptReport} for ApplicationAttemptId + * @throws YarnException * @throws IOException */ @Public @Unstable ApplicationAttemptReport getApplicationAttempt( - ApplicationAttemptId appAttemptId) throws IOException; + ApplicationAttemptId appAttemptId) throws YarnException, IOException; /** * This method returns {@link ContainerReport} for specified @@ -92,11 +99,13 @@ ApplicationAttemptReport getApplicationAttempt( * @param containerId * {@link ContainerId} * @return {@link ContainerReport} for ContainerId + * @throws YarnException * @throws IOException */ @Public @Unstable - ContainerReport getContainer(ContainerId containerId) throws IOException; + ContainerReport getContainer(ContainerId containerId) + throws YarnException, IOException; /** * This method returns {@link ContainerReport} for specified @@ -105,12 +114,13 @@ ApplicationAttemptReport getApplicationAttempt( * @param appAttemptId * {@link ApplicationAttemptId} * @return {@link ContainerReport} for ApplicationAttemptId + * @throws YarnException * @throws IOException */ @Public @Unstable ContainerReport getAMContainer(ApplicationAttemptId appAttemptId) - throws IOException; + throws YarnException, IOException; /** * This method returns Map of {@link ContainerId} to {@link ContainerReport} @@ -120,10 +130,11 @@ ContainerReport getAMContainer(ApplicationAttemptId appAttemptId) * {@link ApplicationAttemptId} * @return Map of {@link ContainerId} to {@link ContainerReport} for * ApplicationAttemptId + * @throws YarnException * @throws IOException */ @Public @Unstable Map getContainers( - ApplicationAttemptId appAttemptId) throws IOException; + ApplicationAttemptId appAttemptId) throws YarnException, IOException; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java index f6a40bd..ee34c49 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java @@ -34,6 +34,9 @@ public static final String FINISHED_EVENT_TYPE = "YARN_APPLICATION_FINISHED"; + public static final String ACLS_UPDATED_EVENT_TYPE = + "YARN_APPLICATION_ACLS_UPDATED"; + public static final String NAME_ENTITY_INFO = "YARN_APPLICATION_NAME"; @@ -49,6 +52,9 @@ public static final String SUBMITTED_TIME_ENTITY_INFO = "YARN_APPLICATION_SUBMITTED_TIME"; + public static final String APP_VIEW_ACLS_ENTITY_INFO = + "YARN_APPLICATION_VIEW_ACLS"; + public static final String DIAGNOSTICS_INFO_EVENT_INFO = "YARN_APPLICATION_DIAGNOSTICS_INFO"; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java index 4a02892..d4ca660 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java @@ -138,14 +138,14 @@ public ApplicationAttemptReport run() throws Exception { .p() ._( "Sorry, Failed to get containers for application attempt" + attemptid - + ".")._(); + + ": " + e.getMessage())._(); return; } catch (Exception e) { html .p() ._( "Sorry, Failed to get containers for application attempt" + attemptid - + ".")._(); + + ": " + e.getMessage())._(); return; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java index 8fa4086..26f35d6 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java @@ -87,7 +87,8 @@ public ApplicationReport run() throws Exception { }); } } catch (Exception e) { - String message = "Failed to read the application " + appID + "."; + String message = "Failed to read the application " + appID + ": " + + e.getMessage(); LOG.error(message, e); html.p()._(message)._(); return; @@ -131,8 +132,8 @@ public ApplicationReport run() throws Exception { }); } } catch (Exception e) { - String message = - "Failed to read the attempts of the application " + appID + "."; + String message = "Failed to read the attempts of the application " + + appID + ": " + e.getMessage(); LOG.error(message, e); html.p()._(message)._(); return; @@ -165,7 +166,8 @@ public ContainerReport run() throws Exception { } catch (Exception e) { String message = "Failed to read the AM container of the application attempt " - + appAttemptReport.getApplicationAttemptId() + "."; + + appAttemptReport.getApplicationAttemptId() + ": " + + e.getMessage(); LOG.error(message, e); html.p()._(message)._(); return; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java index 19ea5fe..0f312c2 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java @@ -86,7 +86,7 @@ public void render(Block html) { }); } } catch (Exception e) { - String message = "Failed to read the applications."; + String message = "Failed to read the applications: " + e.getMessage(); LOG.error(message, e); html.p()._(message)._(); return; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ContainerBlock.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ContainerBlock.java index 2bb48a8..9243f78 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ContainerBlock.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ContainerBlock.java @@ -80,7 +80,8 @@ public ContainerReport run() throws Exception { }); } } catch (Exception e) { - String message = "Failed to read the container " + containerid + "."; + String message = "Failed to read the container " + containerid + ": " + + e.getMessage(); LOG.error(message, e); html.p()._(message)._(); return; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationACLsUpdatedEvent.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationACLsUpdatedEvent.java new file mode 100644 index 0000000..34e674a --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationACLsUpdatedEvent.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.metrics; + +import java.util.Collection; + +import org.apache.hadoop.yarn.api.records.ApplicationId; + + +public class ApplicationACLsUpdatedEvent extends YarnMetricsEvent { + + private ApplicationId appId; + private Collection viewAppACLs; + + public ApplicationACLsUpdatedEvent(ApplicationId appId, + Collection viewAppACLs, + long updatedTime) { + super(YarnMetricsEventType.APP_ACLS_UPDATED, updatedTime); + this.appId = appId; + this.viewAppACLs = viewAppACLs; + } + + public ApplicationId getApplicationId() { + return appId; + } + + public Collection getViewAppACLs() { + return viewAppACLs; + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/YarnMetricsEventType.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/YarnMetricsEventType.java index c319f6a..99c987f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/YarnMetricsEventType.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/YarnMetricsEventType.java @@ -23,6 +23,7 @@ // app events APP_CREATED, APP_FINISHED, + APP_ACLS_UPDATED, // app attempt events APP_ATTEMPT_REGISTERED, diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/YarnMetricsPublisher.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/YarnMetricsPublisher.java index 0d1d7bb..b07b3e2 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/YarnMetricsPublisher.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/YarnMetricsPublisher.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.metrics; import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -52,6 +53,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; +import org.apache.hadoop.yarn.util.StringHelper; @Private @Unstable @@ -125,6 +127,18 @@ public void appFinished(RMApp app, RMAppState state, long finishedTime) { } @SuppressWarnings("unchecked") + public void appACLsUpdated(RMApp app, Collection appViewACLs, + long updatedTime) { + if (publishYarnMetrics) { + dispatcher.getEventHandler().handle( + new ApplicationACLsUpdatedEvent( + app.getApplicationId(), + appViewACLs, + updatedTime)); + } + } + + @SuppressWarnings("unchecked") public void appAttemptRegistered(RMAppAttempt appAttempt, long registeredTime) { if (publishYarnMetrics) { @@ -201,6 +215,9 @@ protected void handleYarnMetricsEvent( case APP_FINISHED: publishApplicationFinishedEvent((ApplicationFinishedEvent) event); break; + case APP_ACLS_UPDATED: + publishApplicationACLsUpdatedEvent((ApplicationACLsUpdatedEvent) event); + break; case APP_ATTEMPT_REGISTERED: publishAppAttemptRegisteredEvent((AppAttemptRegisteredEvent) event); break; @@ -261,6 +278,22 @@ private void publishApplicationFinishedEvent(ApplicationFinishedEvent event) { putEntity(entity); } + private void publishApplicationACLsUpdatedEvent( + ApplicationACLsUpdatedEvent event) { + TimelineEntity entity = + createApplicationEntity(event.getApplicationId()); + TimelineEvent tEvent = new TimelineEvent(); + Map entityInfo = new HashMap(); + entityInfo.put(ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO, + StringHelper.joins(":", event.getViewAppACLs().toArray())); + entity.setOtherInfo(entityInfo); + tEvent.setEventType( + ApplicationMetricsConstants.ACLS_UPDATED_EVENT_TYPE); + tEvent.setTimestamp(event.getTimestamp()); + entity.addEvent(tEvent); + putEntity(entity); + } + private static TimelineEntity createApplicationEntity( ApplicationId applicationId) { TimelineEntity entity = new TimelineEntity(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestYarnMetricsPublisher.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestYarnMetricsPublisher.java index 5dae022..1a1af08 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestYarnMetricsPublisher.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestYarnMetricsPublisher.java @@ -21,6 +21,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import java.util.Arrays; import java.util.EnumSet; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; @@ -99,14 +100,15 @@ public void testPublishApplicationMetrics() throws Exception { RMApp app = createRMApp(appId); metricsPublisher.appCreated(app, app.getStartTime()); metricsPublisher.appFinished(app, RMAppState.FINISHED, app.getFinishTime()); + metricsPublisher.appACLsUpdated(app, Arrays.asList("uers1,user2"), 4L); TimelineEntity entity = null; do { entity = store.getEntity(appId.toString(), ApplicationMetricsConstants.ENTITY_TYPE, EnumSet.allOf(Field.class)); - // ensure two events are both published before leaving the loop - } while (entity == null || entity.getEvents().size() < 2); + // ensure three events are both published before leaving the loop + } while (entity == null || entity.getEvents().size() < 3); // verify all the fields Assert.assertEquals(ApplicationMetricsConstants.ENTITY_TYPE, entity.getEntityType()); @@ -133,8 +135,12 @@ public void testPublishApplicationMetrics() throws Exception { Assert.assertEquals(app.getSubmitTime(), entity.getOtherInfo().get( ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO)); + Assert.assertEquals("uers1,user2", + entity.getOtherInfo().get( + ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO)); boolean hasCreatedEvent = false; boolean hasFinishedEvent = false; + boolean hasACLsUpdatedEvent = false; for (TimelineEvent event : entity.getEvents()) { if (event.getEventType().equals( ApplicationMetricsConstants.CREATED_EVENT_TYPE)) { @@ -154,9 +160,13 @@ public void testPublishApplicationMetrics() throws Exception { ApplicationMetricsConstants.FINAL_STATUS_EVENT_INFO)); Assert.assertEquals(YarnApplicationState.FINISHED.toString(), event .getEventInfo().get(ApplicationMetricsConstants.STATE_EVENT_INFO)); + } else if (event.getEventType().equals( + ApplicationMetricsConstants.ACLS_UPDATED_EVENT_TYPE)) { + hasACLsUpdatedEvent = true; + Assert.assertEquals(4L, event.getTimestamp()); } } - Assert.assertTrue(hasCreatedEvent && hasFinishedEvent); + Assert.assertTrue(hasCreatedEvent && hasFinishedEvent && hasACLsUpdatedEvent); } @Test(timeout = 10000)