diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java index 44f57f4..4a73007 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java @@ -344,6 +344,9 @@ public static final String DEFAULT_JOB_ACL_MODIFY_JOB = " "; + public static final String JOB_SKIP_RM_TOKEN_RENEWAL = + "mapreduce.job.skip.rm.token.renewal"; + /* config for tracking the local file where all the credentials for the job * credentials. */ diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml index 4535137..1d83f6f 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml @@ -1005,6 +1005,17 @@ + mapreduce.job.skip.rm.token.renewal + false + + *Note*: Advanced property. Change with caution. + When this property is set to true, Resouce Manager will skip renewing job + token at application submission. See YARN-3021 for a scenario that doing + so is needed. + + + + mapreduce.job.acl-view-job Job specific access-control list for 'viewing' the job. It is diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java index 41dc72f..6da6e9f 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java @@ -529,6 +529,8 @@ public ApplicationSubmissionContext createApplicationSubmissionContext( if (tagsFromConf != null && !tagsFromConf.isEmpty()) { appContext.setApplicationTags(new HashSet(tagsFromConf)); } + appContext.setSkipRMTokenRenewal( + jobConf.getBoolean(MRJobConfig.JOB_SKIP_RM_TOKEN_RENEWAL, false)); return appContext; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java index f1ebbfe..64a95b6 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java @@ -75,7 +75,7 @@ public static ApplicationSubmissionContext newInstance( boolean isUnmanagedAM, boolean cancelTokensWhenComplete, int maxAppAttempts, Resource resource, String applicationType, boolean keepContainers, String appLabelExpression, - String amContainerLabelExpression) { + String amContainerLabelExpression, boolean skipRMTokenRenewal) { ApplicationSubmissionContext context = Records.newRecord(ApplicationSubmissionContext.class); context.setApplicationId(applicationId); @@ -86,10 +86,10 @@ public static ApplicationSubmissionContext newInstance( context.setUnmanagedAM(isUnmanagedAM); context.setCancelTokensWhenComplete(cancelTokensWhenComplete); context.setMaxAppAttempts(maxAppAttempts); + context.setResource(resource); context.setApplicationType(applicationType); context.setKeepContainersAcrossApplicationAttempts(keepContainers); context.setNodeLabelExpression(appLabelExpression); - context.setResource(resource); ResourceRequest amReq = Records.newRecord(ResourceRequest.class); amReq.setResourceName(ResourceRequest.ANY); @@ -98,8 +98,26 @@ public static ApplicationSubmissionContext newInstance( amReq.setRelaxLocality(true); amReq.setNodeLabelExpression(amContainerLabelExpression); context.setAMContainerResourceRequest(amReq); + + context.setSkipRMTokenRenewal(skipRMTokenRenewal); + return context; } + + @Public + @Stable + public static ApplicationSubmissionContext newInstance( + ApplicationId applicationId, String applicationName, String queue, + Priority priority, ContainerLaunchContext amContainer, + boolean isUnmanagedAM, boolean cancelTokensWhenComplete, + int maxAppAttempts, Resource resource, String applicationType, + boolean keepContainers, String appLabelExpression, + String amContainerLabelExpression) { + return newInstance(applicationId, applicationName, queue, priority, + amContainer, isUnmanagedAM, cancelTokensWhenComplete, maxAppAttempts, + resource, applicationType, keepContainers, appLabelExpression, + amContainerLabelExpression, false); + } public static ApplicationSubmissionContext newInstance( ApplicationId applicationId, String applicationName, String queue, @@ -474,6 +492,21 @@ public abstract void setKeepContainersAcrossApplicationAttempts( public abstract void setAMContainerResourceRequest(ResourceRequest request); /** + * @return true if to skip token renewal at ResourceManager. + */ + @LimitedPrivate("mapreduce") + @Evolving + public abstract boolean getSkipRMTokenRenewal(); + + /** + * Set skipRMTokenRenewal field of ApplicationSubmissionContext + * @param boolean skipTokenRenewal + */ + @Public + @Evolving + public abstract void setSkipRMTokenRenewal(boolean skipRMTokenRenewal); + + /** * Get the attemptFailuresValidityInterval in milliseconds for the application * * @return the attemptFailuresValidityInterval diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index c4e756d..0d2f808 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -303,6 +303,7 @@ message ApplicationSubmissionContextProto { optional ReservationIdProto reservation_id = 15; optional string node_label_expression = 16; optional ResourceRequestProto am_container_resource_request = 17; + optional bool skip_rm_token_renewal = 18 [default = false]; } message LogAggregationContextProto { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java index 303b437..d771086 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java @@ -496,6 +496,18 @@ public void setAMContainerResourceRequest(ResourceRequest request) { } @Override + public boolean getSkipRMTokenRenewal() { + ApplicationSubmissionContextProtoOrBuilder p = viaProto ? proto : builder; + return p.getSkipRmTokenRenewal(); + } + + @Override + public void setSkipRMTokenRenewal(boolean skipRMTokenRenewal) { + maybeInitBuilder(); + builder.setSkipRmTokenRenewal(skipRMTokenRenewal); + } + + @Override public long getAttemptFailuresValidityInterval() { ApplicationSubmissionContextProtoOrBuilder p = viaProto ? proto : builder; return p.getAttemptFailuresValidityInterval(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java index b8f6e9c..2dfe90b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java @@ -664,6 +664,18 @@ public void testApplicationSubmissionContextPBImpl() throws Exception { } @Test + public void testApplicationSubmissionContextPBImpl1() throws Exception { + validatePBImplRecord(ApplicationSubmissionContextPBImpl.class, + ApplicationSubmissionContextProto.class); + + ApplicationSubmissionContext ctx = + ApplicationSubmissionContext.newInstance(null, null, null, null, null, + false, false, 0, Resources.none(), null, false, null, null, true); + + Assert.assertNotNull(ctx.getResource()); + } + + @Test @Ignore // ignore cause ApplicationIdPBImpl is immutable public void testContainerIdPBImpl() throws Exception { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index f38e128..095ff3a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -281,7 +281,8 @@ protected void submitApplication( createAndPopulateNewRMApp(submissionContext, submitTime, user); ApplicationId appId = submissionContext.getApplicationId(); - if (UserGroupInformation.isSecurityEnabled()) { + if (UserGroupInformation.isSecurityEnabled() && + !submissionContext.getSkipRMTokenRenewal()) { try { this.rmContext.getDelegationTokenRenewer().addApplicationAsync(appId, parseCredentials(submissionContext), diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index 48b4a3c..ecb0179 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -1265,7 +1265,8 @@ protected ApplicationSubmissionContext createAppSubmissionContext( newApp.getApplicationType(), newApp.getKeepContainersAcrossApplicationAttempts(), newApp.getAppNodeLabelExpression(), - newApp.getAMContainerNodeLabelExpression()); + newApp.getAMContainerNodeLabelExpression(), + newApp.getSkipRMTokenRenewal()); appContext.setApplicationTags(newApp.getApplicationTags()); return appContext; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationSubmissionContextInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationSubmissionContextInfo.java index 5278b3e..a8e0906 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationSubmissionContextInfo.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationSubmissionContextInfo.java @@ -78,6 +78,9 @@ @XmlElement(name = "am-container-node-label-expression") String amContainerNodeLabelExpression; + @XmlElement(name = "skip-rm-token-renewal") + boolean skipRMTokenRenewal; + public ApplicationSubmissionContextInfo() { applicationId = ""; applicationName = ""; @@ -91,6 +94,7 @@ public ApplicationSubmissionContextInfo() { tags = new HashSet(); appNodeLabelExpression = ""; amContainerNodeLabelExpression = ""; + skipRMTokenRenewal = false; } public String getApplicationId() { @@ -149,6 +153,10 @@ public String getAMContainerNodeLabelExpression() { return amContainerNodeLabelExpression; } + public boolean getSkipRMTokenRenewal() { + return skipRMTokenRenewal; + } + public void setApplicationId(String applicationId) { this.applicationId = applicationId; } @@ -206,4 +214,8 @@ public void setAppNodeLabelExpression(String appNodeLabelExpression) { public void setAMContainerNodeLabelExpression(String nodeLabelExpression) { this.amContainerNodeLabelExpression = nodeLabelExpression; } + + public void setSkipRMTokenRenewal(boolean skipRMTokenRenewal) { + this.skipRMTokenRenewal = skipRMTokenRenewal; + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java index 5d31404..38b833b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java @@ -808,8 +808,8 @@ public void run() { submitThread.join(); } - @Test(timeout=20000) - public void testAppSubmissionWithInvalidDelegationToken() throws Exception { + private void testAppSubmissionWithInvalidDelegationTokenHelper( + boolean skip_rm_token_renewal) throws Exception { Configuration conf = new Configuration(); conf.set( CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, @@ -831,18 +831,35 @@ protected void doSecureLogin() throws IOException { ApplicationSubmissionContext.newInstance( ApplicationId.newInstance(1234121, 0), "BOGUS", "default", Priority.UNDEFINED, amContainer, false, - true, 1, Resource.newInstance(1024, 1), "BOGUS"); + true, 1, Resource.newInstance(1024, 1), "BOGUS", + false, null, null, skip_rm_token_renewal); SubmitApplicationRequest request = SubmitApplicationRequest.newInstance(appSubContext); try { rm.getClientRMService().submitApplication(request); - fail("Error was excepted."); + if (!skip_rm_token_renewal) { + fail("Error was excepted."); + } } catch (YarnException e) { - Assert.assertTrue(e.getMessage().contains( - "Bad header found in token storage")); + if (!skip_rm_token_renewal) { + Assert.assertTrue(e.getMessage().contains( + "Bad header found in token storage")); + } else { + throw e; + } } } + @Test(timeout=20000) + public void testAppSubmissionWithInvalidDelegationToken() throws Exception { + testAppSubmissionWithInvalidDelegationTokenHelper(false); + } + + @Test(timeout=20000) + public void testAppSubmissionWithInvalidDelegationTokenSkipRmTokenRenewal() + throws Exception { + testAppSubmissionWithInvalidDelegationTokenHelper(true); + } @Test (timeout = 20000) public void testReplaceExpiringDelegationToken() throws Exception {