diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java index 32bbe96..95cf75d 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java @@ -1013,4 +1013,6 @@ * A comma-separated list of properties whose value will be redacted. */ String MR_JOB_REDACTED_PROPERTIES = "mapreduce.job.redacted-properties"; + + String MR_JOB_CONF_SENT = "mapreduce.jobconf.sent"; } diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java index 4c6f0f3..0b69fa4 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java @@ -498,6 +498,10 @@ public ApplicationSubmissionContext createApplicationSubmissionContext( ContainerLaunchContext amContainer = ContainerLaunchContext.newInstance(localResources, environment, vargsFinal, null, securityTokens, acls); + if (UserGroupInformation.isSecurityEnabled() && conf + .getBoolean(MRJobConfig.MR_JOB_CONF_SENT, false)) { + setAppConf(amContainer, conf); + } Collection tagsFromConf = jobConf.getTrimmedStringCollection(MRJobConfig.JOB_TAGS); @@ -576,6 +580,15 @@ public ApplicationSubmissionContext createApplicationSubmissionContext( return appContext; } + private void setAppConf(ContainerLaunchContext context, Configuration conf) + throws IOException { + DataOutputBuffer dob = new DataOutputBuffer(); + Configuration copy = new Configuration(conf); + copy.write(dob); + ByteBuffer appConf = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); + context.setTokensConf(appConf); + } + @Override public void setJobPriority(JobID arg0, String arg1) throws IOException, InterruptedException { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerLaunchContext.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerLaunchContext.java index 6d4bccd..616aa4b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerLaunchContext.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerLaunchContext.java @@ -108,6 +108,22 @@ public static ContainerLaunchContext newInstance( public abstract void setTokens(ByteBuffer tokens); /** + * Get the configuration used by RM to renew tokens. + * @return The configuration used by RM to renew the tokens. + */ + @Public + @Unstable + public abstract ByteBuffer getTokensConf(); + + /** + * Set the configuration used by RM to renew the tokens. + * @param tokensConf The configuration used by RM to renew the tokens + */ + @Public + @Unstable + public abstract void setTokensConf(ByteBuffer tokensConf); + + /** * Get LocalResource required by the container. * @return all LocalResource required by the container */ diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 5a70298..c805261 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -547,6 +547,8 @@ message ContainerLaunchContextProto { repeated string command = 5; repeated ApplicationACLMapProto application_ACLs = 6; optional ContainerRetryContextProto container_retry_context = 7; + optional bytes tokens_conf = 8; + } message ContainerStatusProto { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java index 1efe541..1f76c34 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java @@ -54,6 +54,7 @@ private Map localResources = null; private ByteBuffer tokens = null; + private ByteBuffer tokensConf = null; private Map serviceData = null; private Map environment = null; private List commands = null; @@ -111,6 +112,9 @@ private void mergeLocalToBuilder() { if (this.tokens != null) { builder.setTokens(convertToProtoFormat(this.tokens)); } + if (this.tokensConf != null) { + builder.setTokensConf(convertToProtoFormat(this.tokensConf)); + } if (this.serviceData != null) { addServiceDataToProto(); } @@ -268,6 +272,28 @@ public void setTokens(ByteBuffer tokens) { } @Override + public ByteBuffer getTokensConf() { + ContainerLaunchContextProtoOrBuilder p = viaProto ? proto : builder; + if (this.tokensConf != null) { + return this.tokensConf; + } + if (!p.hasTokensConf()) { + return null; + } + this.tokensConf = convertFromProtoFormat(p.getTokensConf()); + return this.tokensConf; + } + + @Override + public void setTokensConf(ByteBuffer tokensConf) { + maybeInitBuilder(); + if (tokensConf == null) { + builder.clearTokensConf(); + } + this.tokensConf = tokensConf; + } + + @Override public Map getServiceData() { initServiceData(); return this.serviceData; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java index 74c06ff..ee443de 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java @@ -29,8 +29,11 @@ import java.util.Set; import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.DataInputByteBuffer; import org.apache.hadoop.io.Text; import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; import org.apache.hadoop.yarn.api.records.AMCommand; @@ -496,4 +499,31 @@ public static AllocateResponse newAllocateResponse(int responseId, return response; } + + public static Credentials parseCredentials( + ApplicationSubmissionContext application) throws IOException { + Credentials credentials = new Credentials(); + DataInputByteBuffer dibb = new DataInputByteBuffer(); + ByteBuffer tokens = application.getAMContainerSpec().getTokens(); + if (tokens != null) { + dibb.reset(tokens); + credentials.readTokenStorageStream(dibb); + tokens.rewind(); + } + return credentials; + } + + public static Configuration parseTokensConf( + ApplicationSubmissionContext context) throws IOException { + ByteBuffer tokensConf = context.getAMContainerSpec().getTokensConf(); + if (tokensConf == null) { + return null; + } + DataInputByteBuffer dibb = new DataInputByteBuffer(); + dibb.reset(tokensConf); + Configuration appConf = new Configuration(); + appConf.readFields(dibb); + tokensConf.rewind(); + return appConf; + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index 3dc7e38..85fb7bf 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -607,6 +607,15 @@ public SubmitApplicationResponse submitApplication( } } + if (!UserGroupInformation.isSecurityEnabled() && + request.getApplicationSubmissionContext().getAMContainerSpec() + .getTokensConf() != null) { + throw new YarnException( + "Token conf does not need to be set in non-secure environment for " + + submissionContext.getApplicationId()); + + } + // Check whether app has already been put into rmContext, // If it is, simply return the response if (rmContext.getRMApps().get(applicationId) != null) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index 4d628ee..8579d69 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -17,18 +17,14 @@ */ package org.apache.hadoop.yarn.server.resourcemanager; -import java.io.IOException; -import java.nio.ByteBuffer; import java.util.LinkedList; import java.util.Map; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.io.DataInputByteBuffer; import org.apache.hadoop.ipc.Server; import org.apache.hadoop.security.AccessControlException; -import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; @@ -297,14 +293,14 @@ protected void submitApplication( // constructor. RMAppImpl application = createAndPopulateNewRMApp( submissionContext, submitTime, user, false, -1); - Credentials credentials = null; try { - credentials = parseCredentials(submissionContext); if (UserGroupInformation.isSecurityEnabled()) { this.rmContext.getDelegationTokenRenewer() - .addApplicationAsync(applicationId, credentials, + .addApplicationAsync(applicationId, + BuilderUtils.parseCredentials(submissionContext), submissionContext.getCancelTokensWhenComplete(), - application.getUser()); + application.getUser(), + BuilderUtils.parseTokensConf(submissionContext)); } else { // Dispatcher is not yet started at this time, so these START events // enqueued should be guaranteed to be first processed when dispatcher @@ -466,20 +462,7 @@ private ResourceRequest validateAndCreateResourceRequest( return null; } - - protected Credentials parseCredentials( - ApplicationSubmissionContext application) throws IOException { - Credentials credentials = new Credentials(); - DataInputByteBuffer dibb = new DataInputByteBuffer(); - ByteBuffer tokens = application.getAMContainerSpec().getTokens(); - if (tokens != null) { - dibb.reset(tokens); - credentials.readTokenStorageStream(dibb); - tokens.rewind(); - } - return credentials; - } - + @Override public void recover(RMState state) throws Exception { RMStateStore store = rmContext.getStateStore(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index a647969..08a2594 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -1103,9 +1103,12 @@ public RMAppState transition(RMAppImpl app, RMAppEvent event) { try { app.rmContext.getDelegationTokenRenewer() .addApplicationAsyncDuringRecovery(app.getApplicationId(), - app.parseCredentials(), + BuilderUtils.parseCredentials(app.submissionContext), app.submissionContext.getCancelTokensWhenComplete(), - app.getUser()); + app.getUser(), + BuilderUtils.parseTokensConf(app.submissionContext)); + // set the memory free + app.submissionContext.getAMContainerSpec().setTokensConf(null); } catch (Exception e) { String msg = "Failed to fetch user credentials from application:" + e.getMessage(); @@ -1158,6 +1161,8 @@ public void transition(RMAppImpl app, RMAppEvent event) { app.submissionContext, false, app.applicationPriority)); // send the ATS create Event app.sendATSCreateEvent(); + // Set the memory free after submission context is persisted + app.submissionContext.getAMContainerSpec().setTokensConf(null); } } @@ -1473,6 +1478,8 @@ public void transition(RMAppImpl app, RMAppEvent event) { .applicationFinished(app, finalState); app.rmContext.getSystemMetricsPublisher() .appFinished(app, finalState, app.finishTime); + // set the memory free + app.submissionContext.getAMContainerSpec().setTokensConf(null); }; } @@ -1682,18 +1689,6 @@ public ResourceRequest getAMResourceRequest() { return this.amReq; } - protected Credentials parseCredentials() throws IOException { - Credentials credentials = new Credentials(); - DataInputByteBuffer dibb = new DataInputByteBuffer(); - ByteBuffer tokens = submissionContext.getAMContainerSpec().getTokens(); - if (tokens != null) { - dibb.reset(tokens); - credentials.readTokenStorageStream(dibb); - tokens.rewind(); - } - return credentials; - } - @Override public Map getLogAggregationReportsForApp() { try { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java index dfbf333..4d51579 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java @@ -379,43 +379,44 @@ public Void run() throws Exception { * @param applicationId added application * @param ts tokens * @param shouldCancelAtEnd true if tokens should be canceled when the app is - * done else false. + * done else false. * @param user user + * @param appConf */ public void addApplicationAsync(ApplicationId applicationId, Credentials ts, - boolean shouldCancelAtEnd, String user) { + boolean shouldCancelAtEnd, String user, Configuration appConf) { processDelegationTokenRenewerEvent(new DelegationTokenRenewerAppSubmitEvent( - applicationId, ts, shouldCancelAtEnd, user)); + applicationId, ts, shouldCancelAtEnd, user, appConf)); } /** * Asynchronously add application tokens for renewal. - * - * @param applicationId + * @param applicationId * added application * @param ts * tokens * @param shouldCancelAtEnd - * true if tokens should be canceled when the app is done else false. + * true if tokens should be canceled when the app is done else false. * @param user - * user + * @param appConf */ public void addApplicationAsyncDuringRecovery(ApplicationId applicationId, - Credentials ts, boolean shouldCancelAtEnd, String user) { + Credentials ts, boolean shouldCancelAtEnd, String user, + Configuration appConf) { processDelegationTokenRenewerEvent( new DelegationTokenRenewerAppRecoverEvent(applicationId, ts, - shouldCancelAtEnd, user)); + shouldCancelAtEnd, user, appConf)); } /** + * Only for testing * Synchronously renew delegation tokens. - * @param user user */ public void addApplicationSync(ApplicationId applicationId, Credentials ts, boolean shouldCancelAtEnd, String user) throws IOException, InterruptedException { handleAppSubmitEvent(new DelegationTokenRenewerAppSubmitEvent( - applicationId, ts, shouldCancelAtEnd, user)); + applicationId, ts, shouldCancelAtEnd, user, new Configuration())); } private void handleAppSubmitEvent(AbstractDelegationTokenRenewerAppEvent evt) @@ -455,8 +456,18 @@ private void handleAppSubmitEvent(AbstractDelegationTokenRenewerAppEvent evt) DelegationTokenToRenew dttr = allTokens.get(token); if (dttr == null) { + Configuration appConf; + if (evt.appConf != null) { + appConf = new Configuration(getConfig()); + // Override conf with app provided conf - this is required in cases + // where RM does not have the required conf to communicate with + // remote cluster. The conf is provided by the application itself. + appConf.addResource(evt.appConf); + } else { + appConf = getConfig(); + } dttr = new DelegationTokenToRenew(Arrays.asList(applicationId), token, - getConfig(), now, shouldCancelAtEnd, evt.getUser()); + appConf, now, shouldCancelAtEnd, evt.getUser()); try { renewToken(dttr); } catch (IOException ioe) { @@ -926,22 +937,22 @@ private void handleDTRenewerAppRecoverEvent( } static class DelegationTokenRenewerAppSubmitEvent - extends - AbstractDelegationTokenRenewerAppEvent { + extends AbstractDelegationTokenRenewerAppEvent { public DelegationTokenRenewerAppSubmitEvent(ApplicationId appId, - Credentials credentails, boolean shouldCancelAtEnd, String user) { + Credentials credentails, boolean shouldCancelAtEnd, String user, + Configuration appConf) { super(appId, credentails, shouldCancelAtEnd, user, - DelegationTokenRenewerEventType.VERIFY_AND_START_APPLICATION); + DelegationTokenRenewerEventType.VERIFY_AND_START_APPLICATION, appConf); } } static class DelegationTokenRenewerAppRecoverEvent - extends - AbstractDelegationTokenRenewerAppEvent { + extends AbstractDelegationTokenRenewerAppEvent { public DelegationTokenRenewerAppRecoverEvent(ApplicationId appId, - Credentials credentails, boolean shouldCancelAtEnd, String user) { + Credentials credentails, boolean shouldCancelAtEnd, String user, + Configuration appConf) { super(appId, credentails, shouldCancelAtEnd, user, - DelegationTokenRenewerEventType.RECOVER_APPLICATION); + DelegationTokenRenewerEventType.RECOVER_APPLICATION, appConf); } } @@ -949,16 +960,18 @@ public DelegationTokenRenewerAppRecoverEvent(ApplicationId appId, DelegationTokenRenewerEvent { private Credentials credentials; + private Configuration appConf; private boolean shouldCancelAtEnd; private String user; public AbstractDelegationTokenRenewerAppEvent(ApplicationId appId, - Credentials credentails, boolean shouldCancelAtEnd, String user, - DelegationTokenRenewerEventType type) { + Credentials credentials, boolean shouldCancelAtEnd, String user, + DelegationTokenRenewerEventType type, Configuration appConf) { super(appId, type); - this.credentials = credentails; + this.credentials = credentials; this.shouldCancelAtEnd = shouldCancelAtEnd; this.user = user; + this.appConf = appConf; } public Credentials getCredentials() { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java index 9223ef3..61e4437 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java @@ -110,6 +110,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; +import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorContext; import org.apache.hadoop.yarn.server.utils.BuilderUtils; @@ -1732,26 +1733,16 @@ public void testAppFailedOnSubmissionSavedInStateStore() throws Exception { memStore.init(conf); MockRM rm1 = new TestSecurityMockRM(conf, memStore) { - @Override - protected RMAppManager createRMAppManager() { - return new TestRMAppManager(this.rmContext, this.scheduler, - this.masterService, this.applicationACLsManager, conf); - } - - class TestRMAppManager extends RMAppManager { - - public TestRMAppManager(RMContext context, YarnScheduler scheduler, - ApplicationMasterService masterService, - ApplicationACLsManager applicationACLsManager, Configuration conf) { - super(context, scheduler, masterService, applicationACLsManager, conf); - } - - @Override - protected Credentials parseCredentials( - ApplicationSubmissionContext application) throws IOException { - throw new IOException("Parsing credential error."); + class TestDelegationTokenRenewer extends DelegationTokenRenewer { + public void addApplicationAsync(ApplicationId applicationId, Credentials ts, + boolean shouldCancelAtEnd, String user, Configuration appConf) { + throw new RuntimeException("failed to submit app"); } } + @Override + protected DelegationTokenRenewer createDelegationTokenRenewer() { + return new TestDelegationTokenRenewer(); + } }; rm1.start(); RMApp app1 = diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java index 205188b..4487e1d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java @@ -392,7 +392,8 @@ public void testDTRenewal () throws Exception { // register the tokens for renewal ApplicationId applicationId_0 = BuilderUtils.newApplicationId(0, 0); - delegationTokenRenewer.addApplicationAsync(applicationId_0, ts, true, "user"); + delegationTokenRenewer.addApplicationAsync(applicationId_0, ts, true, "user", + new Configuration()); waitForEventsToGetProcessed(delegationTokenRenewer); // first 3 initial renewals + 1 real @@ -432,7 +433,8 @@ public void testDTRenewal () throws Exception { ApplicationId applicationId_1 = BuilderUtils.newApplicationId(0, 1); - delegationTokenRenewer.addApplicationAsync(applicationId_1, ts, true, "user"); + delegationTokenRenewer.addApplicationAsync(applicationId_1, ts, true, "user", + new Configuration()); waitForEventsToGetProcessed(delegationTokenRenewer); delegationTokenRenewer.applicationFinished(applicationId_1); waitForEventsToGetProcessed(delegationTokenRenewer); @@ -468,7 +470,8 @@ public void testAppRejectionWithCancelledDelegationToken() throws Exception { // register the tokens for renewal ApplicationId appId = BuilderUtils.newApplicationId(0, 0); - delegationTokenRenewer.addApplicationAsync(appId, ts, true, "user"); + delegationTokenRenewer.addApplicationAsync(appId, ts, true, "user", + new Configuration()); int waitCnt = 20; while (waitCnt-- >0) { if (!eventQueue.isEmpty()) { @@ -531,7 +534,8 @@ public void testDTRenewalWithNoCancel () throws Exception { ApplicationId applicationId_1 = BuilderUtils.newApplicationId(0, 1); - delegationTokenRenewer.addApplicationAsync(applicationId_1, ts, false, "user"); + delegationTokenRenewer.addApplicationAsync(applicationId_1, ts, false, "user", + new Configuration()); waitForEventsToGetProcessed(delegationTokenRenewer); delegationTokenRenewer.applicationFinished(applicationId_1); waitForEventsToGetProcessed(delegationTokenRenewer); @@ -600,7 +604,8 @@ public void testDTKeepAlive1 () throws Exception { // register the tokens for renewal ApplicationId applicationId_0 = BuilderUtils.newApplicationId(0, 0); - localDtr.addApplicationAsync(applicationId_0, ts, true, "user"); + localDtr.addApplicationAsync(applicationId_0, ts, true, "user", + new Configuration()); waitForEventsToGetProcessed(localDtr); if (!eventQueue.isEmpty()){ Event evt = eventQueue.take(); @@ -679,7 +684,8 @@ public void testDTKeepAlive2() throws Exception { // register the tokens for renewal ApplicationId applicationId_0 = BuilderUtils.newApplicationId(0, 0); - localDtr.addApplicationAsync(applicationId_0, ts, true, "user"); + localDtr.addApplicationAsync(applicationId_0, ts, true, "user", + new Configuration()); localDtr.applicationFinished(applicationId_0); waitForEventsToGetProcessed(delegationTokenRenewer); //Send another keep alive. @@ -831,14 +837,16 @@ public Long answer(InvocationOnMock invocation) Thread submitThread = new Thread() { @Override public void run() { - dtr.addApplicationAsync(mock(ApplicationId.class), creds1, false, "user"); + dtr.addApplicationAsync(mock(ApplicationId.class), creds1, false, "user", + new Configuration()); } }; submitThread.start(); // wait till 1st submit blocks, then submit another startBarrier.await(); - dtr.addApplicationAsync(mock(ApplicationId.class), creds2, false, "user"); + dtr.addApplicationAsync(mock(ApplicationId.class), creds2, false, "user", + new Configuration()); // signal 1st to complete endBarrier.await(); submitThread.join();