From bece4dd49998217e9a3c9cf35893a6efb5fdcfcd Mon Sep 17 00:00:00 2001 From: Sunil G Date: Mon, 11 Sep 2017 13:56:29 +0530 Subject: [PATCH] YARN-7157 --- .../apache/hadoop/yarn/conf/YarnConfiguration.java | 8 +++ .../server/resourcemanager/ClientRMService.java | 25 ++++++- .../resourcemanager/TestClientRMService.java | 80 +++++++++++++++++++++- 3 files changed, 107 insertions(+), 6 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index be63233..5504cf0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -3106,6 +3106,14 @@ public static boolean areNodeLabelsEnabled( public static final String NM_SCRIPT_BASED_NODE_LABELS_PROVIDER_SCRIPT_OPTS = NM_SCRIPT_BASED_NODE_LABELS_PROVIDER_PREFIX + "opts"; + /* + * Support to view apps for given user in secure cluster. + */ + public static final String DISPLAY_APPS_FOR_LOGGED_IN_USER = + RM_PREFIX + "display.per-user-apps"; + public static final boolean DEFAULT_DISPLAY_APPS_FOR_LOGGED_IN_USER = + false; + // RM and NM CSRF props public static final String REST_CSRF = "webapp.rest-csrf."; public static final String RM_CSRF_PREFIX = RM_PREFIX + REST_CSRF; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index df38893..dcdf8cc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -142,6 +142,7 @@ import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.ipc.RPCUtil; import org.apache.hadoop.yarn.ipc.YarnRPC; +import org.apache.hadoop.yarn.security.YarnAuthorizationProvider; import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; @@ -207,6 +208,8 @@ private ReservationSystem reservationSystem; private ReservationInputValidator rValidator; + private boolean displayPerUserApps = false; + private static final EnumSet ACTIVE_APP_STATES = EnumSet.of( RMAppState.ACCEPTED, RMAppState.RUNNING); @@ -264,7 +267,11 @@ protected void serviceStart() throws Exception { } refreshServiceAcls(conf, RMPolicyProvider.getInstance()); } - + + this.displayPerUserApps = conf.getBoolean( + YarnConfiguration.DISPLAY_APPS_FOR_LOGGED_IN_USER, + YarnConfiguration.DEFAULT_DISPLAY_APPS_FOR_LOGGED_IN_USER); + this.server.start(); clientBindAddress = conf.updateConnectAddr(YarnConfiguration.RM_BIND_HOST, YarnConfiguration.RM_ADDRESS, @@ -861,8 +868,8 @@ public void remove() { } } - if (users != null && !users.isEmpty() && - !users.contains(application.getUser())) { + if (users != null && !users.isEmpty() + && !users.contains(application.getUser())) { continue; } @@ -898,6 +905,12 @@ public void remove() { continue; } + // Given RM is configured to display apps per user, skip apps to which + // this caller doesn't have access to view. + if (displayPerUserApps && !allowAccess) { + continue; + } + reports.add(application.createAndGetApplicationReport( callerUGI.getUserName(), allowAccess)); } @@ -908,6 +921,8 @@ public void remove() { return response; } + + private Set getLowerCasedAppTypes(GetApplicationsRequest request) { Set applicationTypes = new HashSet<>(); if (request.getApplicationTypes() != null && !request.getApplicationTypes() @@ -1766,4 +1781,8 @@ private RMApp verifyUserAccessForRMApp(ApplicationId applicationId, return application; } + @VisibleForTesting + public void setDisplayPerUserApps(boolean displayPerUserApps) { + this.displayPerUserApps = displayPerUserApps; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index ea733a4..cd0d022 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -1054,9 +1054,9 @@ public void handle(Event event) {} for (int i = 0; i < appIds.length; i++) { ApplicationId appId = appIds[i]; when(mockAclsManager.checkAccess(UserGroupInformation.getCurrentUser(), - ApplicationAccessType.VIEW_APP, null, appId)).thenReturn(true); - SubmitApplicationRequest submitRequest = mockSubmitAppRequest( - appId, appNames[i], queues[i % queues.length], + ApplicationAccessType.VIEW_APP, null, appId)).thenReturn(true); + SubmitApplicationRequest submitRequest = mockSubmitAppRequest(appId, + appNames[i], queues[i % queues.length], new HashSet(tags.subList(0, i + 1))); rmService.submitApplication(submitRequest); submitTimeMillis[i] = System.currentTimeMillis(); @@ -1118,6 +1118,12 @@ public void handle(Event event) {} assertEquals("Incorrect number of applications for user", 3, rmService.getApplications(request).getApplicationList().size()); + rmService.setDisplayPerUserApps(true); + userSet.clear(); + assertEquals("Incorrect number of applications for user", 6, + rmService.getApplications(request).getApplicationList().size()); + rmService.setDisplayPerUserApps(false); + // Check tags request = GetApplicationsRequest.newInstance( ApplicationsRequestScope.ALL, null, null, null, null, null, null, @@ -2061,4 +2067,72 @@ protected ClientRMService createClientRMService() { rpc.stopProxy(client, conf); new File(excludeFile).delete(); } + + @Test + public void testGetApplicationsWithPerUserApps() + throws IOException, YarnException { + /* + * Submit 3 applications alternately in two queues + */ + // Basic setup + YarnScheduler yarnScheduler = mockYarnScheduler(); + RMContext rmContext = mock(RMContext.class); + mockRMContext(yarnScheduler, rmContext); + RMStateStore stateStore = mock(RMStateStore.class); + when(rmContext.getStateStore()).thenReturn(stateStore); + doReturn(mock(RMTimelineCollectorManager.class)).when(rmContext) + .getRMTimelineCollectorManager(); + + RMAppManager appManager = new RMAppManager(rmContext, yarnScheduler, null, + mock(ApplicationACLsManager.class), new Configuration()); + when(rmContext.getDispatcher().getEventHandler()) + .thenReturn(new EventHandler() { + public void handle(Event event) { + } + }); + + // Simulate Queue ACL manager which returns false always + QueueACLsManager queueAclsManager = mock(QueueACLsManager.class); + when(queueAclsManager.checkAccess(any(UserGroupInformation.class), + any(QueueACL.class), any(RMApp.class), any(String.class), + anyListOf(String.class))).thenReturn(false); + + // Simulate app ACL manager which returns false always + ApplicationACLsManager appAclsManager = mock(ApplicationACLsManager.class); + when(appAclsManager.checkAccess(eq(UserGroupInformation.getCurrentUser()), + any(ApplicationAccessType.class), any(String.class), + any(ApplicationId.class))).thenReturn(false); + ClientRMService rmService = new ClientRMService(rmContext, yarnScheduler, + appManager, appAclsManager, queueAclsManager, null); + rmService.init(new Configuration()); + + // Initialize appnames and queues + String[] queues = {QUEUE_1, QUEUE_2}; + String[] appNames = {MockApps.newAppName(), MockApps.newAppName(), + MockApps.newAppName()}; + ApplicationId[] appIds = {getApplicationId(101), getApplicationId(102), + getApplicationId(103)}; + List tags = Arrays.asList("Tag1", "Tag2", "Tag3"); + + long[] submitTimeMillis = new long[3]; + // Submit applications + for (int i = 0; i < appIds.length; i++) { + ApplicationId appId = appIds[i]; + SubmitApplicationRequest submitRequest = mockSubmitAppRequest(appId, + appNames[i], queues[i % queues.length], + new HashSet(tags.subList(0, i + 1))); + rmService.submitApplication(submitRequest); + submitTimeMillis[i] = System.currentTimeMillis(); + } + + // Test different cases of ClientRMService#getApplications() + GetApplicationsRequest request = GetApplicationsRequest.newInstance(); + assertEquals("Incorrect total number of apps", 6, + rmService.getApplications(request).getApplicationList().size()); + + rmService.setDisplayPerUserApps(true); + assertEquals("Incorrect number of applications for user", 0, + rmService.getApplications(request).getApplicationList().size()); + rmService.setDisplayPerUserApps(false); + } } -- 2.10.1 (Apple Git-78)