diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index c375887476..26a5043f67 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -195,7 +195,7 @@ protected RMDelegationTokenSecretManager rmDTSecretManager; private final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); - InetSocketAddress clientBindAddress; + private InetSocketAddress clientBindAddress; private final ApplicationACLsManager applicationsACLsManager; private final QueueACLsManager queueACLsManager; @@ -205,9 +205,6 @@ private ReservationSystem reservationSystem; private ReservationInputValidator rValidator; - private static final EnumSet COMPLETED_APP_STATES = EnumSet.of( - RMAppState.FINISHED, RMAppState.FINISHING, RMAppState.FAILED, - RMAppState.KILLED, RMAppState.FINAL_SAVING, RMAppState.KILLING); private static final EnumSet ACTIVE_APP_STATES = EnumSet.of( RMAppState.ACCEPTED, RMAppState.RUNNING); @@ -297,11 +294,12 @@ public InetSocketAddress getBindAddress() { /** * check if the calling user has the access to application information. - * @param callerUGI - * @param owner - * @param operationPerformed - * @param application - * @return + * @param callerUGI the user information who submit the request + * @param owner the user of the application + * @param operationPerformed the type of operation defined in + * {@link ApplicationAccessType} + * @param application submitted application + * @return access is permitted or not */ private boolean checkAccess(UserGroupInformation callerUGI, String owner, ApplicationAccessType operationPerformed, RMApp application) { @@ -377,23 +375,17 @@ public GetApplicationReportResponse getApplicationReport( public GetApplicationAttemptReportResponse getApplicationAttemptReport( GetApplicationAttemptReportRequest request) throws YarnException, IOException { - ApplicationAttemptId appAttemptId = request.getApplicationAttemptId(); - UserGroupInformation callerUGI; - try { - callerUGI = UserGroupInformation.getCurrentUser(); - } catch (IOException ie) { - LOG.info("Error getting UGI ", ie); - throw RPCUtil.getRemoteException(ie); - } - RMApp application = this.rmContext.getRMApps().get( - appAttemptId.getApplicationId()); - if (application == null) { - // If the RM doesn't have the application, throw - // ApplicationNotFoundException and let client to handle. - throw new ApplicationNotFoundException("Application with id '" - + request.getApplicationAttemptId().getApplicationId() - + "' doesn't exist in RM."); + ApplicationId applicationId + = request.getApplicationAttemptId().getApplicationId(); + if (applicationId == null) { + throw new ApplicationNotFoundException("Invalid application id: null"); } + ApplicationAttemptId appAttemptId = request.getApplicationAttemptId(); + + UserGroupInformation callerUGI = getCallerUgi(applicationId, + AuditConstants.GET_APP_ATTEMPT_REPORT); + RMApp application = verifyUserAccessForRMApp(applicationId, callerUGI, + AuditConstants.GET_APP_ATTEMPT_REPORT); boolean allowAccess = checkAccess(callerUGI, application.getUser(), ApplicationAccessType.VIEW_APP, application); @@ -419,20 +411,11 @@ public GetApplicationAttemptReportResponse getApplicationAttemptReport( public GetApplicationAttemptsResponse getApplicationAttempts( GetApplicationAttemptsRequest request) throws YarnException, IOException { ApplicationId appId = request.getApplicationId(); - UserGroupInformation callerUGI; - try { - callerUGI = UserGroupInformation.getCurrentUser(); - } catch (IOException ie) { - LOG.info("Error getting UGI ", ie); - throw RPCUtil.getRemoteException(ie); - } - RMApp application = this.rmContext.getRMApps().get(appId); - if (application == null) { - // If the RM doesn't have the application, throw - // ApplicationNotFoundException and let client to handle. - throw new ApplicationNotFoundException("Application with id '" + appId - + "' doesn't exist in RM."); - } + UserGroupInformation callerUGI = getCallerUgi(appId, + AuditConstants.GET_APP_ATTEMPTS); + RMApp application = verifyUserAccessForRMApp(appId, callerUGI, + AuditConstants.GET_APP_ATTEMPTS); + boolean allowAccess = checkAccess(callerUGI, application.getUser(), ApplicationAccessType.VIEW_APP, application); GetApplicationAttemptsResponse response = null; @@ -467,20 +450,11 @@ public GetContainerReportResponse getContainerReport( ContainerId containerId = request.getContainerId(); ApplicationAttemptId appAttemptId = containerId.getApplicationAttemptId(); ApplicationId appId = appAttemptId.getApplicationId(); - UserGroupInformation callerUGI; - try { - callerUGI = UserGroupInformation.getCurrentUser(); - } catch (IOException ie) { - LOG.info("Error getting UGI ", ie); - throw RPCUtil.getRemoteException(ie); - } - RMApp application = this.rmContext.getRMApps().get(appId); - if (application == null) { - // If the RM doesn't have the application, throw - // ApplicationNotFoundException and let client to handle. - throw new ApplicationNotFoundException("Application with id '" + appId - + "' doesn't exist in RM."); - } + UserGroupInformation callerUGI = getCallerUgi(appId, + AuditConstants.GET_CONTAINER_REPORT); + RMApp application = verifyUserAccessForRMApp(appId, callerUGI, + AuditConstants.GET_CONTAINER_REPORT); + boolean allowAccess = checkAccess(callerUGI, application.getUser(), ApplicationAccessType.VIEW_APP, application); GetContainerReportResponse response = null; @@ -491,13 +465,13 @@ public GetContainerReportResponse getContainerReport( "ApplicationAttempt with id '" + appAttemptId + "' doesn't exist in RM."); } - RMContainer rmConatiner = this.rmContext.getScheduler().getRMContainer( + RMContainer rmContainer = this.rmContext.getScheduler().getRMContainer( containerId); - if (rmConatiner == null) { + if (rmContainer == null) { throw new ContainerNotFoundException("Container with id '" + containerId + "' doesn't exist in RM."); } - response = GetContainerReportResponse.newInstance(rmConatiner + response = GetContainerReportResponse.newInstance(rmContainer .createContainerReport()); } else { throw new YarnException("User " + callerUGI.getShortUserName() @@ -517,20 +491,11 @@ public GetContainersResponse getContainers(GetContainersRequest request) throws YarnException, IOException { ApplicationAttemptId appAttemptId = request.getApplicationAttemptId(); ApplicationId appId = appAttemptId.getApplicationId(); - UserGroupInformation callerUGI; - try { - callerUGI = UserGroupInformation.getCurrentUser(); - } catch (IOException ie) { - LOG.info("Error getting UGI ", ie); - throw RPCUtil.getRemoteException(ie); - } - RMApp application = this.rmContext.getRMApps().get(appId); - if (application == null) { - // If the RM doesn't have the application, throw - // ApplicationNotFoundException and let client to handle. - throw new ApplicationNotFoundException("Application with id '" + appId - + "' doesn't exist in RM."); - } + UserGroupInformation callerUGI = getCallerUgi(appId, + AuditConstants.GET_CONTAINERS); + RMApp application = verifyUserAccessForRMApp(appId, callerUGI, + AuditConstants.GET_CONTAINERS); + boolean allowAccess = checkAccess(callerUGI, application.getUser(), ApplicationAccessType.VIEW_APP, application); GetContainersResponse response = null; @@ -594,6 +559,7 @@ public SubmitApplicationResponse submitApplication( TimelineUtils.FLOW_RUN_ID_TAG_PREFIX.toLowerCase() + ":")) { value = tag.substring(TimelineUtils.FLOW_RUN_ID_TAG_PREFIX.length() + 1); + // In order to check the number format Long.valueOf(value); } } @@ -656,9 +622,8 @@ public SubmitApplicationResponse submitApplication( throw e; } - SubmitApplicationResponse response = recordFactory + return recordFactory .newRecordInstance(SubmitApplicationResponse.class); - return response; } @SuppressWarnings("unchecked") @@ -669,26 +634,10 @@ public FailApplicationAttemptResponse failApplicationAttempt( ApplicationAttemptId attemptId = request.getApplicationAttemptId(); ApplicationId applicationId = attemptId.getApplicationId(); - UserGroupInformation callerUGI; - try { - callerUGI = UserGroupInformation.getCurrentUser(); - } catch (IOException ie) { - LOG.info("Error getting UGI ", ie); - RMAuditLogger.logFailure("UNKNOWN", AuditConstants.FAIL_ATTEMPT_REQUEST, - "UNKNOWN", "ClientRMService" , "Error getting UGI", - applicationId, attemptId); - throw RPCUtil.getRemoteException(ie); - } - - RMApp application = this.rmContext.getRMApps().get(applicationId); - if (application == null) { - RMAuditLogger.logFailure(callerUGI.getUserName(), - AuditConstants.FAIL_ATTEMPT_REQUEST, "UNKNOWN", "ClientRMService", - "Trying to fail an attempt of an absent application", applicationId, - attemptId); - throw new ApplicationNotFoundException("Trying to fail an attempt " - + attemptId + " of an absent application " + applicationId); - } + UserGroupInformation callerUGI = getCallerUgi(applicationId, + AuditConstants.FAIL_ATTEMPT_REQUEST); + RMApp application = verifyUserAccessForRMApp(applicationId, callerUGI, + AuditConstants.FAIL_ATTEMPT_REQUEST); RMAppAttempt appAttempt = application.getAppAttempts().get(attemptId); if (appAttempt == null) { @@ -696,28 +645,14 @@ public FailApplicationAttemptResponse failApplicationAttempt( "ApplicationAttempt with id '" + attemptId + "' doesn't exist in RM."); } - if (!checkAccess(callerUGI, application.getUser(), - ApplicationAccessType.MODIFY_APP, application)) { - RMAuditLogger.logFailure(callerUGI.getShortUserName(), - AuditConstants.FAIL_ATTEMPT_REQUEST, - "User doesn't have permissions to " - + ApplicationAccessType.MODIFY_APP.toString(), "ClientRMService", - AuditConstants.UNAUTHORIZED_USER, applicationId); - throw RPCUtil.getRemoteException(new AccessControlException("User " - + callerUGI.getShortUserName() + " cannot perform operation " - + ApplicationAccessType.MODIFY_APP.name() + " on " + applicationId)); - } - FailApplicationAttemptResponse response = recordFactory.newRecordInstance(FailApplicationAttemptResponse.class); - if (!ACTIVE_APP_STATES.contains(application.getState())) { - if (COMPLETED_APP_STATES.contains(application.getState())) { - RMAuditLogger.logSuccess(callerUGI.getShortUserName(), - AuditConstants.FAIL_ATTEMPT_REQUEST, "ClientRMService", - applicationId); - return response; - } + if (application.isAppInCompletedStates()) { + RMAuditLogger.logSuccess(callerUGI.getShortUserName(), + AuditConstants.FAIL_ATTEMPT_REQUEST, "ClientRMService", + applicationId); + return response; } this.rmContext.getDispatcher().getEventHandler().handle( @@ -745,11 +680,10 @@ public KillApplicationResponse forceKillApplication( } catch (IOException ie) { LOG.info("Error getting UGI ", ie); RMAuditLogger.logFailure("UNKNOWN", AuditConstants.KILL_APP_REQUEST, - "UNKNOWN", "ClientRMService" , "Error getting UGI", - applicationId, callerContext); + "UNKNOWN", "ClientRMService", "Error getting UGI", + applicationId, callerContext); throw RPCUtil.getRemoteException(ie); } - RMApp application = this.rmContext.getRMApps().get(applicationId); if (application == null) { RMAuditLogger.logFailure(callerUGI.getUserName(), @@ -795,7 +729,7 @@ public KillApplicationResponse forceKillApplication( .handle(new RMAppKillByClientEvent(applicationId, message.toString(), callerUGI, remoteAddress)); - // For UnmanagedAMs, return true so they don't retry + // For Unmanaged AMs, return true so they don't retry return KillApplicationResponse.newInstance( application.getApplicationSubmissionContext().getUnmanagedAM()); } @@ -1087,15 +1021,15 @@ public GetDelegationTokenResponse getDelegationToken( RMDelegationTokenIdentifier tokenIdentifier = new RMDelegationTokenIdentifier(owner, new Text(request.getRenewer()), realUser); - Token realRMDTtoken = + Token realRMDToken = new Token(tokenIdentifier, this.rmDTSecretManager); response.setRMDelegationToken( BuilderUtils.newDelegationToken( - realRMDTtoken.getIdentifier(), - realRMDTtoken.getKind().toString(), - realRMDTtoken.getPassword(), - realRMDTtoken.getService().toString() + realRMDToken.getIdentifier(), + realRMDToken.getKind().toString(), + realRMDToken.getPassword(), + realRMDToken.getService().toString() )); return response; } catch(IOException io) { @@ -1155,37 +1089,10 @@ public MoveApplicationAcrossQueuesResponse moveApplicationAcrossQueues( MoveApplicationAcrossQueuesRequest request) throws YarnException { ApplicationId applicationId = request.getApplicationId(); - UserGroupInformation callerUGI; - try { - callerUGI = UserGroupInformation.getCurrentUser(); - } catch (IOException ie) { - LOG.info("Error getting UGI ", ie); - RMAuditLogger.logFailure("UNKNOWN", AuditConstants.MOVE_APP_REQUEST, - "UNKNOWN", "ClientRMService" , "Error getting UGI", - applicationId); - throw RPCUtil.getRemoteException(ie); - } - - RMApp application = this.rmContext.getRMApps().get(applicationId); - if (application == null) { - RMAuditLogger.logFailure(callerUGI.getUserName(), - AuditConstants.MOVE_APP_REQUEST, "UNKNOWN", "ClientRMService", - "Trying to move an absent application", applicationId); - throw new ApplicationNotFoundException("Trying to move an absent" - + " application " + applicationId); - } - - if (!checkAccess(callerUGI, application.getUser(), - ApplicationAccessType.MODIFY_APP, application)) { - RMAuditLogger.logFailure(callerUGI.getShortUserName(), - AuditConstants.MOVE_APP_REQUEST, - "User doesn't have permissions to " - + ApplicationAccessType.MODIFY_APP.toString(), "ClientRMService", - AuditConstants.UNAUTHORIZED_USER, applicationId); - throw RPCUtil.getRemoteException(new AccessControlException("User " - + callerUGI.getShortUserName() + " cannot perform operation " - + ApplicationAccessType.MODIFY_APP.name() + " on " + applicationId)); - } + UserGroupInformation callerUGI = getCallerUgi(applicationId, + AuditConstants.MOVE_APP_REQUEST); + RMApp application = verifyUserAccessForRMApp(applicationId, callerUGI, + AuditConstants.MOVE_APP_REQUEST); String targetQueue = request.getTargetQueue(); if (!accessToTargetQueueAllowed(callerUGI, application, targetQueue)) { @@ -1224,9 +1131,8 @@ public MoveApplicationAcrossQueuesResponse moveApplicationAcrossQueues( RMAuditLogger.logSuccess(callerUGI.getShortUserName(), AuditConstants.MOVE_APP_REQUEST, "ClientRMService" , applicationId); - MoveApplicationAcrossQueuesResponse response = recordFactory + return recordFactory .newRecordInstance(MoveApplicationAcrossQueuesResponse.class); - return response; } /** @@ -1283,7 +1189,7 @@ public Server getServer() { @Override public GetNewReservationResponse getNewReservation( GetNewReservationRequest request) throws YarnException, IOException { - checkReservationSytem(AuditConstants.CREATE_NEW_RESERVATION_REQUEST); + checkReservationSystem(AuditConstants.CREATE_NEW_RESERVATION_REQUEST); GetNewReservationResponse response = recordFactory.newRecordInstance(GetNewReservationResponse.class); @@ -1297,7 +1203,7 @@ public GetNewReservationResponse getNewReservation( public ReservationSubmissionResponse submitReservation( ReservationSubmissionRequest request) throws YarnException, IOException { // Check if reservation system is enabled - checkReservationSytem(AuditConstants.SUBMIT_RESERVATION_REQUEST); + checkReservationSystem(AuditConstants.SUBMIT_RESERVATION_REQUEST); ReservationSubmissionResponse response = recordFactory.newRecordInstance(ReservationSubmissionResponse.class); ReservationId reservationId = request.getReservationId(); @@ -1356,7 +1262,7 @@ public ReservationSubmissionResponse submitReservation( public ReservationUpdateResponse updateReservation( ReservationUpdateRequest request) throws YarnException, IOException { // Check if reservation system is enabled - checkReservationSytem(AuditConstants.UPDATE_RESERVATION_REQUEST); + checkReservationSystem(AuditConstants.UPDATE_RESERVATION_REQUEST); ReservationUpdateResponse response = recordFactory.newRecordInstance(ReservationUpdateResponse.class); // Validate the input @@ -1395,7 +1301,7 @@ public ReservationUpdateResponse updateReservation( public ReservationDeleteResponse deleteReservation( ReservationDeleteRequest request) throws YarnException, IOException { // Check if reservation system is enabled - checkReservationSytem(AuditConstants.DELETE_RESERVATION_REQUEST); + checkReservationSystem(AuditConstants.DELETE_RESERVATION_REQUEST); ReservationDeleteResponse response = recordFactory.newRecordInstance(ReservationDeleteResponse.class); // Validate the input @@ -1434,7 +1340,7 @@ public ReservationDeleteResponse deleteReservation( public ReservationListResponse listReservations( ReservationListRequest requestInfo) throws YarnException, IOException { // Check if reservation system is enabled - checkReservationSytem(AuditConstants.LIST_RESERVATION_REQUEST); + checkReservationSystem(AuditConstants.LIST_RESERVATION_REQUEST); ReservationListResponse response = recordFactory.newRecordInstance(ReservationListResponse.class); @@ -1474,9 +1380,7 @@ public ReservationListResponse listReservations( public GetNodesToLabelsResponse getNodeToLabels( GetNodesToLabelsRequest request) throws YarnException, IOException { RMNodeLabelsManager labelsMgr = rmContext.getNodeLabelManager(); - GetNodesToLabelsResponse response = - GetNodesToLabelsResponse.newInstance(labelsMgr.getNodeLabelsInfo()); - return response; + return GetNodesToLabelsResponse.newInstance(labelsMgr.getNodeLabelsInfo()); } @Override @@ -1496,13 +1400,12 @@ public GetLabelsToNodesResponse getLabelsToNodes( public GetClusterNodeLabelsResponse getClusterNodeLabels( GetClusterNodeLabelsRequest request) throws YarnException, IOException { RMNodeLabelsManager labelsMgr = rmContext.getNodeLabelManager(); - GetClusterNodeLabelsResponse response = - GetClusterNodeLabelsResponse.newInstance( + return GetClusterNodeLabelsResponse.newInstance( labelsMgr.getClusterNodeLabels()); - return response; } - private void checkReservationSytem(String auditConstant) throws YarnException { + private void checkReservationSystem(String auditConstant) + throws YarnException { // Check if reservation is enabled if (reservationSystem == null) { throw RPCUtil.getRemoteException("Reservation is not enabled." @@ -1663,9 +1566,14 @@ public UpdateApplicationPriorityResponse updateApplicationPriority( } /** - * Signal a container. + * Send a signal to a container. + * * After the request passes some sanity check, it will be delivered * to RMNodeImpl so that the next NM heartbeat will pick up the signal request + * @param request request to signal a container + * @return the response of sending signal request + * @throws YarnException rpc related exception + * @throws IOException fail to obtain user group information */ @SuppressWarnings("unchecked") @Override @@ -1756,7 +1664,7 @@ public UpdateApplicationTimeoutsResponse updateApplicationTimeouts( if (!EnumSet .of(RMAppState.SUBMITTED, RMAppState.ACCEPTED, RMAppState.RUNNING) .contains(state)) { - if (COMPLETED_APP_STATES.contains(state)) { + if (application.isAppInCompletedStates()) { // If Application is in any of the final states, update timeout // can be skipped rather throwing exception. RMAuditLogger.logSuccess(callerUGI.getShortUserName(), diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java index 051d979e12..fc0b265580 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java @@ -54,6 +54,11 @@ public static final String GET_APP_STATE = "Get Application State"; public static final String GET_APP_PRIORITY = "Get Application Priority"; public static final String GET_APP_QUEUE = "Get Application Queue"; + public static final String GET_APP_ATTEMPTS = "Get Application Attempts"; + public static final String GET_APP_ATTEMPT_REPORT + = "Get Application Attempt Report"; + public static final String GET_CONTAINERS = "Get Containers"; + public static final String GET_CONTAINER_REPORT = "Get Container Report"; public static final String FINISH_SUCCESS_APP = "Application Finished - Succeeded"; public static final String FINISH_FAILED_APP = "Application Finished - Failed"; public static final String FINISH_KILLED_APP = "Application Finished - Killed";