diff --git hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/Token.java hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/Token.java index 32a8cd3..31edcec 100644 --- hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/Token.java +++ hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/Token.java @@ -40,7 +40,8 @@ @InterfaceStability.Evolving public class Token implements Writable { public static final Log LOG = LogFactory.getLog(Token.class); - + public static final String NON_RENEWER = ""; + private static Map> tokenKindMap; private byte[] identifier; diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java index f0a6ddf..46ac73c 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java @@ -362,6 +362,8 @@ public static final String JOB_NAMENODES = "mapreduce.job.hdfs-servers"; + public static final String JOB_NAMENODES_TOKEN_RENEWAL_EXCLUDE = "mapreduce.job.hdfs-servers.token-renewal.exclude"; + public static final String JOB_JOBTRACKER_ID = "mapreduce.job.kerberos.jtprinicipal"; public static final String JOB_CANCEL_DELEGATION_TOKEN = "mapreduce.job.complete.cancel.delegation.tokens"; diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java index 7b1f657..6c45d3c 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java @@ -101,6 +101,20 @@ static void obtainTokensForNamenodesInternal(Credentials credentials, } } + static boolean isTokenRenewalExcluded(FileSystem fs, Configuration conf) { + String [] nns = + conf.getStrings(MRJobConfig.JOB_NAMENODES_TOKEN_RENEWAL_EXCLUDE); + if(nns!= null) { + String host = fs.getUri().getHost(); + for(int i=0; i< nns.length; i++) { + if (nns[i].equals(host)) { + return true; + } + } + } + return false; + } + /** * get delegation token for a specific FS * @param fs @@ -111,11 +125,15 @@ static void obtainTokensForNamenodesInternal(Credentials credentials, */ static void obtainTokensForNamenodesInternal(FileSystem fs, Credentials credentials, Configuration conf) throws IOException { - String delegTokenRenewer = Master.getMasterPrincipal(conf); - if (delegTokenRenewer == null || delegTokenRenewer.length() == 0) { - throw new IOException( - "Can't get Master Kerberos principal for use as renewer"); + String delegTokenRenewer = Token.NON_RENEWER; + if (!isTokenRenewalExcluded(fs, conf)) { + delegTokenRenewer = Master.getMasterPrincipal(conf); + if (delegTokenRenewer == null || delegTokenRenewer.length() == 0) { + throw new IOException( + "Can't get Master Kerberos principal for use as renewer"); + } } + mergeBinaryTokens(credentials, conf); final Token tokens[] = fs.addDelegationTokens(delegTokenRenewer, diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java index cb456d8..bff12481 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java @@ -437,7 +437,9 @@ private void handleAppSubmitEvent(DelegationTokenRenewerAppSubmitEvent evt) // renewal. for (DelegationTokenToRenew dtr : tokenList) { try { - renewToken(dtr); + if (!skipTokenRenewal(dtr)) { + renewToken(dtr); + } } catch (IOException ioe) { throw new IOException("Failed to renew token: " + dtr.token, ioe); } @@ -445,7 +447,9 @@ private void handleAppSubmitEvent(DelegationTokenRenewerAppSubmitEvent evt) for (DelegationTokenToRenew dtr : tokenList) { appTokens.get(applicationId).add(dtr); allTokens.put(dtr.token, dtr); - setTimerForTokenRenewal(dtr); + if (!skipTokenRenewal(dtr)) { + setTimerForTokenRenewal(dtr); + } } } @@ -479,8 +483,10 @@ public void run() { requestNewHdfsDelegationTokenIfNeeded(dttr); // if the token is not replaced by a new token, renew the token if (appTokens.get(dttr.applicationId).contains(dttr)) { - renewToken(dttr); - setTimerForTokenRenewal(dttr);// set the next one + if (!skipTokenRenewal(dttr)) { + renewToken(dttr); + setTimerForTokenRenewal(dttr);// set the next one + } } else { LOG.info("The token was removed already. Token = [" +dttr +"]"); } @@ -496,14 +502,25 @@ public boolean cancel() { return super.cancel(); } } - + + /* + * Skip renewing token if the renewer of the token is set to + * Token.NON_RENEWER + */ + private boolean skipTokenRenewal(DelegationTokenToRenew token) + throws IOException { + @SuppressWarnings("unchecked") + Text renewer = ((Token)token.token). + decodeIdentifier().getRenewer(); + return (renewer != null && renewer.toString().equals(Token.NON_RENEWER)); + } + /** * set task to renew the token */ @VisibleForTesting protected void setTimerForTokenRenewal(DelegationTokenToRenew token) throws IOException { - // calculate timer time long expiresIn = token.expirationDate - System.currentTimeMillis(); long renewIn = token.expirationDate - expiresIn/10; // little bit before the expiration @@ -592,8 +609,10 @@ private void requestNewHdfsDelegationToken(ApplicationId applicationId, new DelegationTokenToRenew(applicationId, token, getConfig(), Time.now(), shouldCancelAtEnd, user); // renew the token to get the next expiration date. - renewToken(tokenToRenew); - setTimerForTokenRenewal(tokenToRenew); + if (!skipTokenRenewal(tokenToRenew)) { + renewToken(tokenToRenew); + setTimerForTokenRenewal(tokenToRenew); + } appTokens.get(applicationId).add(tokenToRenew); LOG.info("Received new token " + token); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java index 5d31404..069e112 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java @@ -467,7 +467,25 @@ public void testAppRejectionWithCancelledDelegationToken() throws Exception { } fail("App submission with a cancelled token should have failed"); } - + + @Test(timeout=60000) + public void testAppTokenWithNonRenewer() throws Exception { + MyFS dfs = (MyFS)FileSystem.get(conf); + LOG.info("dfs="+(Object)dfs.hashCode() + ";conf="+conf.hashCode()); + + // Test would fail if using "InvalidRenewer" instead of + // Token.NON_RENEWER here + MyToken token = dfs.getDelegationToken(Token.NON_RENEWER); + token.cancelToken(); + + Credentials ts = new Credentials(); + ts.addToken(token.getKind(), token); + + // register the tokens for renewal + ApplicationId appId = BuilderUtils.newApplicationId(0, 0); + delegationTokenRenewer.addApplicationSync(appId, ts, true, "user"); + } + /** * Basic idea of the test: * 1. register a token for 2 seconds with no cancel at the end