diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
index 44f57f4..4a73007 100644
--- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
+++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
@@ -344,6 +344,9 @@
public static final String DEFAULT_JOB_ACL_MODIFY_JOB = " ";
+ public static final String JOB_SKIP_RM_TOKEN_RENEWAL =
+ "mapreduce.job.skip.rm.token.renewal";
+
/* config for tracking the local file where all the credentials for the job
* credentials.
*/
diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
index 4535137..1d83f6f 100644
--- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
+++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
@@ -1005,6 +1005,17 @@
+ mapreduce.job.skip.rm.token.renewal
+ false
+
+ *Note*: Advanced property. Change with caution.
+ When this property is set to true, Resouce Manager will skip renewing job
+ token at application submission. See YARN-3021 for a scenario that doing
+ so is needed.
+
+
+
+
mapreduce.job.acl-view-job
Job specific access-control list for 'viewing' the job. It is
diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
index 41dc72f..6da6e9f 100644
--- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
+++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
@@ -529,6 +529,8 @@ public ApplicationSubmissionContext createApplicationSubmissionContext(
if (tagsFromConf != null && !tagsFromConf.isEmpty()) {
appContext.setApplicationTags(new HashSet(tagsFromConf));
}
+ appContext.setSkipRMTokenRenewal(
+ jobConf.getBoolean(MRJobConfig.JOB_SKIP_RM_TOKEN_RENEWAL, false));
return appContext;
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
index f1ebbfe..64a95b6 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
@@ -75,7 +75,7 @@ public static ApplicationSubmissionContext newInstance(
boolean isUnmanagedAM, boolean cancelTokensWhenComplete,
int maxAppAttempts, Resource resource, String applicationType,
boolean keepContainers, String appLabelExpression,
- String amContainerLabelExpression) {
+ String amContainerLabelExpression, boolean skipRMTokenRenewal) {
ApplicationSubmissionContext context =
Records.newRecord(ApplicationSubmissionContext.class);
context.setApplicationId(applicationId);
@@ -86,10 +86,10 @@ public static ApplicationSubmissionContext newInstance(
context.setUnmanagedAM(isUnmanagedAM);
context.setCancelTokensWhenComplete(cancelTokensWhenComplete);
context.setMaxAppAttempts(maxAppAttempts);
+ context.setResource(resource);
context.setApplicationType(applicationType);
context.setKeepContainersAcrossApplicationAttempts(keepContainers);
context.setNodeLabelExpression(appLabelExpression);
- context.setResource(resource);
ResourceRequest amReq = Records.newRecord(ResourceRequest.class);
amReq.setResourceName(ResourceRequest.ANY);
@@ -98,8 +98,26 @@ public static ApplicationSubmissionContext newInstance(
amReq.setRelaxLocality(true);
amReq.setNodeLabelExpression(amContainerLabelExpression);
context.setAMContainerResourceRequest(amReq);
+
+ context.setSkipRMTokenRenewal(skipRMTokenRenewal);
+
return context;
}
+
+ @Public
+ @Stable
+ public static ApplicationSubmissionContext newInstance(
+ ApplicationId applicationId, String applicationName, String queue,
+ Priority priority, ContainerLaunchContext amContainer,
+ boolean isUnmanagedAM, boolean cancelTokensWhenComplete,
+ int maxAppAttempts, Resource resource, String applicationType,
+ boolean keepContainers, String appLabelExpression,
+ String amContainerLabelExpression) {
+ return newInstance(applicationId, applicationName, queue, priority,
+ amContainer, isUnmanagedAM, cancelTokensWhenComplete, maxAppAttempts,
+ resource, applicationType, keepContainers, appLabelExpression,
+ amContainerLabelExpression, false);
+ }
public static ApplicationSubmissionContext newInstance(
ApplicationId applicationId, String applicationName, String queue,
@@ -474,6 +492,21 @@ public abstract void setKeepContainersAcrossApplicationAttempts(
public abstract void setAMContainerResourceRequest(ResourceRequest request);
/**
+ * @return true if to skip token renewal at ResourceManager.
+ */
+ @LimitedPrivate("mapreduce")
+ @Evolving
+ public abstract boolean getSkipRMTokenRenewal();
+
+ /**
+ * Set skipRMTokenRenewal field of ApplicationSubmissionContext
+ * @param boolean skipTokenRenewal
+ */
+ @Public
+ @Evolving
+ public abstract void setSkipRMTokenRenewal(boolean skipRMTokenRenewal);
+
+ /**
* Get the attemptFailuresValidityInterval in milliseconds for the application
*
* @return the attemptFailuresValidityInterval
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index c4e756d..0d2f808 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -303,6 +303,7 @@ message ApplicationSubmissionContextProto {
optional ReservationIdProto reservation_id = 15;
optional string node_label_expression = 16;
optional ResourceRequestProto am_container_resource_request = 17;
+ optional bool skip_rm_token_renewal = 18 [default = false];
}
message LogAggregationContextProto {
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java
index 303b437..d771086 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java
@@ -496,6 +496,18 @@ public void setAMContainerResourceRequest(ResourceRequest request) {
}
@Override
+ public boolean getSkipRMTokenRenewal() {
+ ApplicationSubmissionContextProtoOrBuilder p = viaProto ? proto : builder;
+ return p.getSkipRmTokenRenewal();
+ }
+
+ @Override
+ public void setSkipRMTokenRenewal(boolean skipRMTokenRenewal) {
+ maybeInitBuilder();
+ builder.setSkipRmTokenRenewal(skipRMTokenRenewal);
+ }
+
+ @Override
public long getAttemptFailuresValidityInterval() {
ApplicationSubmissionContextProtoOrBuilder p = viaProto ? proto : builder;
return p.getAttemptFailuresValidityInterval();
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
index b8f6e9c..2dfe90b 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
@@ -664,6 +664,18 @@ public void testApplicationSubmissionContextPBImpl() throws Exception {
}
@Test
+ public void testApplicationSubmissionContextPBImpl1() throws Exception {
+ validatePBImplRecord(ApplicationSubmissionContextPBImpl.class,
+ ApplicationSubmissionContextProto.class);
+
+ ApplicationSubmissionContext ctx =
+ ApplicationSubmissionContext.newInstance(null, null, null, null, null,
+ false, false, 0, Resources.none(), null, false, null, null, true);
+
+ Assert.assertNotNull(ctx.getResource());
+ }
+
+ @Test
@Ignore
// ignore cause ApplicationIdPBImpl is immutable
public void testContainerIdPBImpl() throws Exception {
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index f38e128..095ff3a 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -281,7 +281,8 @@ protected void submitApplication(
createAndPopulateNewRMApp(submissionContext, submitTime, user);
ApplicationId appId = submissionContext.getApplicationId();
- if (UserGroupInformation.isSecurityEnabled()) {
+ if (UserGroupInformation.isSecurityEnabled() &&
+ !submissionContext.getSkipRMTokenRenewal()) {
try {
this.rmContext.getDelegationTokenRenewer().addApplicationAsync(appId,
parseCredentials(submissionContext),
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
index 48b4a3c..ecb0179 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
@@ -1265,7 +1265,8 @@ protected ApplicationSubmissionContext createAppSubmissionContext(
newApp.getApplicationType(),
newApp.getKeepContainersAcrossApplicationAttempts(),
newApp.getAppNodeLabelExpression(),
- newApp.getAMContainerNodeLabelExpression());
+ newApp.getAMContainerNodeLabelExpression(),
+ newApp.getSkipRMTokenRenewal());
appContext.setApplicationTags(newApp.getApplicationTags());
return appContext;
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationSubmissionContextInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationSubmissionContextInfo.java
index 5278b3e..a8e0906 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationSubmissionContextInfo.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationSubmissionContextInfo.java
@@ -78,6 +78,9 @@
@XmlElement(name = "am-container-node-label-expression")
String amContainerNodeLabelExpression;
+ @XmlElement(name = "skip-rm-token-renewal")
+ boolean skipRMTokenRenewal;
+
public ApplicationSubmissionContextInfo() {
applicationId = "";
applicationName = "";
@@ -91,6 +94,7 @@ public ApplicationSubmissionContextInfo() {
tags = new HashSet();
appNodeLabelExpression = "";
amContainerNodeLabelExpression = "";
+ skipRMTokenRenewal = false;
}
public String getApplicationId() {
@@ -149,6 +153,10 @@ public String getAMContainerNodeLabelExpression() {
return amContainerNodeLabelExpression;
}
+ public boolean getSkipRMTokenRenewal() {
+ return skipRMTokenRenewal;
+ }
+
public void setApplicationId(String applicationId) {
this.applicationId = applicationId;
}
@@ -206,4 +214,8 @@ public void setAppNodeLabelExpression(String appNodeLabelExpression) {
public void setAMContainerNodeLabelExpression(String nodeLabelExpression) {
this.amContainerNodeLabelExpression = nodeLabelExpression;
}
+
+ public void setSkipRMTokenRenewal(boolean skipRMTokenRenewal) {
+ this.skipRMTokenRenewal = skipRMTokenRenewal;
+ }
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java
index 5d31404..38b833b 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java
@@ -808,8 +808,8 @@ public void run() {
submitThread.join();
}
- @Test(timeout=20000)
- public void testAppSubmissionWithInvalidDelegationToken() throws Exception {
+ private void testAppSubmissionWithInvalidDelegationTokenHelper(
+ boolean skip_rm_token_renewal) throws Exception {
Configuration conf = new Configuration();
conf.set(
CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
@@ -831,18 +831,35 @@ protected void doSecureLogin() throws IOException {
ApplicationSubmissionContext.newInstance(
ApplicationId.newInstance(1234121, 0),
"BOGUS", "default", Priority.UNDEFINED, amContainer, false,
- true, 1, Resource.newInstance(1024, 1), "BOGUS");
+ true, 1, Resource.newInstance(1024, 1), "BOGUS",
+ false, null, null, skip_rm_token_renewal);
SubmitApplicationRequest request =
SubmitApplicationRequest.newInstance(appSubContext);
try {
rm.getClientRMService().submitApplication(request);
- fail("Error was excepted.");
+ if (!skip_rm_token_renewal) {
+ fail("Error was excepted.");
+ }
} catch (YarnException e) {
- Assert.assertTrue(e.getMessage().contains(
- "Bad header found in token storage"));
+ if (!skip_rm_token_renewal) {
+ Assert.assertTrue(e.getMessage().contains(
+ "Bad header found in token storage"));
+ } else {
+ throw e;
+ }
}
}
+ @Test(timeout=20000)
+ public void testAppSubmissionWithInvalidDelegationToken() throws Exception {
+ testAppSubmissionWithInvalidDelegationTokenHelper(false);
+ }
+
+ @Test(timeout=20000)
+ public void testAppSubmissionWithInvalidDelegationTokenSkipRmTokenRenewal()
+ throws Exception {
+ testAppSubmissionWithInvalidDelegationTokenHelper(true);
+ }
@Test (timeout = 20000)
public void testReplaceExpiringDelegationToken() throws Exception {