diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java index ac399ec..52e6a01 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java @@ -18,15 +18,8 @@ package org.apache.hadoop.security.token.delegation; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.io.MD5Hash; -import org.apache.hadoop.io.Text; - import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; import java.io.DataInputStream; -import java.io.DataOutputStream; import java.io.IOException; import java.util.Arrays; import java.util.HashMap; @@ -39,10 +32,13 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.io.Text; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.HadoopKerberosName; -import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.SecretManager; +import org.apache.hadoop.security.token.Token; import org.apache.hadoop.util.Daemon; import org.apache.hadoop.util.Time; @@ -386,7 +382,8 @@ private synchronized void removeExpiredKeys() { identifier.setMaxDate(now + tokenMaxLifetime); identifier.setMasterKeyId(currentKey.getKeyId()); identifier.setSequenceNumber(sequenceNum); - LOG.info("Creating password for identifier: [" + MD5Hash.digest(identifier.getBytes()) + ", " + currentKey.getKeyId() + "]"); + LOG.info("Creating password for identifier: " + identifier + + ", currentKey: " + currentKey.getKeyId()); byte[] password = createPassword(identifier.getBytes(), currentKey.getKey()); DelegationTokenInformation tokenInfo = new DelegationTokenInformation(now + tokenRenewInterval, password, getTrackingIdIfEnabled(identifier)); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/YARNDelegationTokenIdentifier.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/YARNDelegationTokenIdentifier.java index 3c5fa867..4a45e28 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/YARNDelegationTokenIdentifier.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/YARNDelegationTokenIdentifier.java @@ -23,192 +23,49 @@ import java.io.IOException; import org.apache.hadoop.io.Text; -import org.apache.hadoop.security.HadoopKerberosName; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier; import org.apache.hadoop.yarn.proto.YarnSecurityTokenProtos.YARNDelegationTokenIdentifierProto; public abstract class YARNDelegationTokenIdentifier extends AbstractDelegationTokenIdentifier { - - YARNDelegationTokenIdentifierProto.Builder builder = - YARNDelegationTokenIdentifierProto.newBuilder(); - public YARNDelegationTokenIdentifier() {} + YARNDelegationTokenIdentifierProto.Builder builder = + YARNDelegationTokenIdentifierProto.newBuilder(); - public YARNDelegationTokenIdentifier(Text owner, Text renewer, Text realUser) { - setOwner(owner); - setRenewer(renewer); - setRealUser(realUser); - } - - /** - * Get the username encoded in the token identifier - * - * @return the username or owner - */ - @Override - public UserGroupInformation getUser() { - String owner = getOwner() == null ? null : getOwner().toString(); - String realUser = getRealUser() == null ? null: getRealUser().toString(); - if ( (owner == null) || (owner.toString().isEmpty())) { - return null; - } - final UserGroupInformation realUgi; - final UserGroupInformation ugi; - if ((realUser == null) || (realUser.toString().isEmpty()) - || realUser.equals(owner)) { - ugi = realUgi = UserGroupInformation.createRemoteUser(owner.toString()); - } else { - realUgi = UserGroupInformation.createRemoteUser(realUser.toString()); - ugi = UserGroupInformation.createProxyUser(owner.toString(), realUgi); - } - realUgi.setAuthenticationMethod(AuthenticationMethod.TOKEN); - return ugi; + public YARNDelegationTokenIdentifier() { } - public Text getOwner() { - String owner = builder.getOwner(); - if (owner == null) { - return null; - } else { - return new Text(owner); - } + public YARNDelegationTokenIdentifier(Text owner, Text renewer, Text realUser) { + super(owner, renewer, realUser); } @Override - public void setOwner(Text owner) { - if (builder != null && owner != null) { - builder.setOwner(owner.toString()); - } - } - - public Text getRenewer() { - String renewer = builder.getRenewer(); - if (renewer == null) { - return null; - } else { - return new Text(renewer); + public void readFields(DataInput in) throws IOException { + builder.mergeFrom((DataInputStream) in); + if (builder.getOwner() != null) { + setOwner(new Text(builder.getOwner())); } - } - - @Override - public void setRenewer(Text renewer) { - if (builder != null && renewer != null) { - HadoopKerberosName renewerKrbName = new HadoopKerberosName(renewer.toString()); - try { - builder.setRenewer(renewerKrbName.getShortName()); - } catch (IOException e) { - throw new RuntimeException(e); - } + if (builder.getRenewer() != null) { + setRenewer(new Text(builder.getRenewer())); } - } - - public Text getRealUser() { - String realUser = builder.getRealUser(); - if (realUser == null) { - return null; - } else { - return new Text(realUser); + if (builder.getRealUser() != null) { + setRealUser(new Text(builder.getRealUser())); } + setIssueDate(builder.getIssueDate()); + setMaxDate(builder.getMaxDate()); + setSequenceNumber(builder.getSequenceNumber()); + setMasterKeyId(builder.getMasterKeyId()); } @Override - public void setRealUser(Text realUser) { - if (builder != null && realUser != null) { - builder.setRealUser(realUser.toString()); - } - } - - public void setIssueDate(long issueDate) { - builder.setIssueDate(issueDate); - } - - public long getIssueDate() { - return builder.getIssueDate(); - } - - - public void setRenewDate(long renewDate) { - builder.setRenewDate(renewDate); - } - - public long getRenewDate() { - return builder.getRenewDate(); - } - - public void setMaxDate(long maxDate) { - builder.setMaxDate(maxDate); - } - - public long getMaxDate() { - return builder.getMaxDate(); - } - - public void setSequenceNumber(int seqNum) { - builder.setSequenceNumber(seqNum); - } - - public int getSequenceNumber() { - return builder.getSequenceNumber(); - } - - public void setMasterKeyId(int newId) { - builder.setMasterKeyId(newId); - } - - public int getMasterKeyId() { - return builder.getMasterKeyId(); - } - - protected static boolean isEqual(Object a, Object b) { - return a == null ? b == null : a.equals(b); - } - - @Override - public boolean equals(Object obj) { - if (obj == this) { - return true; - } - if (obj instanceof YARNDelegationTokenIdentifier) { - YARNDelegationTokenIdentifier that = (YARNDelegationTokenIdentifier) obj; - return this.getSequenceNumber() == that.getSequenceNumber() - && this.getIssueDate() == that.getIssueDate() - && this.getMaxDate() == that.getMaxDate() - && this.getMasterKeyId() == that.getMasterKeyId() - && isEqual(this.getOwner(), that.getOwner()) - && isEqual(this.getRenewer(), that.getRenewer()) - && isEqual(this.getRealUser(), that.getRealUser()); - } - return false; - } - - @Override - public int hashCode() { - return this.getSequenceNumber(); - } - - @Override - public void readFields(DataInput in) throws IOException { - builder.mergeFrom((DataInputStream) in); - } - - @Override public void write(DataOutput out) throws IOException { - builder.build().writeTo((DataOutputStream)out); + builder.setOwner(getOwner().toString()); + builder.setRenewer(getRenewer().toString()); + builder.setRealUser(getRealUser().toString()); + builder.setIssueDate(getIssueDate()); + builder.setMaxDate(getMaxDate()); + builder.setSequenceNumber(getSequenceNumber()); + builder.setMasterKeyId(getMasterKeyId()); + builder.build().writeTo((DataOutputStream) out); } - - @Override - public String toString() { - StringBuilder buffer = new StringBuilder(); - buffer - .append("owner=" + getOwner() + ", renewer=" + getRenewer() + ", realUser=" - + getRealUser() + ", issueDate=" + getIssueDate() - + ", maxDate=" + getMaxDate() + ", sequenceNumber=" - + getSequenceNumber() + ", masterKeyId=" - + getMasterKeyId()); - return buffer.toString(); - } - } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/proto/server/yarn_security_token.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/proto/server/yarn_security_token.proto index 60c7fcc..317032d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/proto/server/yarn_security_token.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/proto/server/yarn_security_token.proto @@ -63,7 +63,6 @@ message YARNDelegationTokenIdentifierProto { optional int64 issueDate = 4; optional int64 maxDate = 5; optional int32 sequenceNumber = 6; - optional int32 masterKeyId = 7 [default = -1]; - optional int64 renewDate = 8; + optional int32 masterKeyId = 7; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/security/TestYARNTokenIdentifier.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/security/TestYARNTokenIdentifier.java index 2e6255a..2052c23 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/security/TestYARNTokenIdentifier.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/security/TestYARNTokenIdentifier.java @@ -257,7 +257,6 @@ public void testTimelineDelegationTokenIdentifier() throws IOException { long maxDate = 2; int sequenceNumber = 3; int masterKeyId = 4; - long renewDate = 5; TimelineDelegationTokenIdentifier token = new TimelineDelegationTokenIdentifier(owner, renewer, realUser); @@ -265,7 +264,6 @@ public void testTimelineDelegationTokenIdentifier() throws IOException { token.setMaxDate(maxDate); token.setSequenceNumber(sequenceNumber); token.setMasterKeyId(masterKeyId); - token.setRenewDate(renewDate); TimelineDelegationTokenIdentifier anotherToken = new TimelineDelegationTokenIdentifier(); @@ -299,9 +297,6 @@ public void testTimelineDelegationTokenIdentifier() throws IOException { Assert.assertEquals("masterKeyId from proto is not the same with original token", anotherToken.getMasterKeyId(), masterKeyId); - - Assert.assertEquals("renewDate from proto is not the same with original token", - anotherToken.getRenewDate(), renewDate); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java index d434e07..51ecaaa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java @@ -369,12 +369,20 @@ private void loadRMDTSecretManagerState(RMState rmState) throws Exception { DelegationKey key = new DelegationKey(); key.readFields(fsIn); rmState.rmSecretManagerState.masterKeyState.add(key); + if (LOG.isDebugEnabled()) { + LOG.debug("Loaded delegation key: keyId=" + key.getKeyId() + + ", expirationDate=" + key.getExpiryDate()); + } } else if (childNodeName.startsWith(DELEGATION_TOKEN_PREFIX)) { RMDelegationTokenIdentifier identifier = new RMDelegationTokenIdentifier(); + long renewDate = fsIn.readLong(); identifier.readFields(fsIn); - long renewDate = identifier.getRenewDate(); rmState.rmSecretManagerState.delegationTokenState.put(identifier, renewDate); + if (LOG.isDebugEnabled()) { + LOG.debug("Loaded RMDelegationTokenIdentifier: " + identifier + + " renewDate=" + renewDate); + } } else { LOG.warn("Unknown file for recovering RMDelegationTokenSecretManager"); } @@ -505,7 +513,7 @@ private void storeOrUpdateRMDelegationTokenAndSequenceNumberState( DELEGATION_TOKEN_PREFIX + identifier.getSequenceNumber()); ByteArrayOutputStream os = new ByteArrayOutputStream(); DataOutputStream fsOut = new DataOutputStream(os); - identifier.setRenewDate(renewDate); + fsOut.writeLong(renewDate); identifier.write(fsOut); if (isUpdate) { LOG.info("Updating RMDelegationToken_" + identifier.getSequenceNumber()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java index fdbf125..0030224 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java @@ -488,6 +488,10 @@ private void loadRMDelegationKeyState(RMState rmState) throws Exception { DelegationKey key = new DelegationKey(); key.readFields(fsIn); rmState.rmSecretManagerState.masterKeyState.add(key); + if (LOG.isDebugEnabled()) { + LOG.debug("Loaded delegation key: keyId=" + key.getKeyId() + + ", expirationDate=" + key.getExpiryDate()); + } } } finally { is.close(); @@ -529,10 +533,14 @@ private void loadRMDelegationTokenState(RMState rmState) throws Exception { if (childNodeName.startsWith(DELEGATION_TOKEN_PREFIX)) { RMDelegationTokenIdentifier identifier = new RMDelegationTokenIdentifier(); + long renewDate = fsIn.readLong(); identifier.readFields(fsIn); - long renewDate = identifier.getRenewDate(); rmState.rmSecretManagerState.delegationTokenState.put(identifier, renewDate); + if (LOG.isDebugEnabled()) { + LOG.debug("Loaded RMDelegationTokenIdentifier: " + identifier + + " renewDate=" + renewDate); + } } } finally { is.close(); @@ -776,7 +784,7 @@ private void addStoreOrUpdateOps(ArrayList opList, DataOutputStream seqOut = new DataOutputStream(seqOs); try { - rmDTIdentifier.setRenewDate(renewDate); + tokenOut.writeLong(renewDate); rmDTIdentifier.write(tokenOut); if (LOG.isDebugEnabled()) { LOG.debug((isUpdate ? "Storing " : "Updating ") + "RMDelegationToken_" + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/RMDelegationTokenIdentifierForTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/RMDelegationTokenIdentifierForTest.java index 5e1baf7..1f44279 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/RMDelegationTokenIdentifierForTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/RMDelegationTokenIdentifierForTest.java @@ -190,6 +190,10 @@ public boolean equals(Object obj) { return false; } + static boolean isEqual(Object a, Object b) { + return a == null ? b == null : a.equals(b); + } + @Override public int hashCode() { return this.getSequenceNumber(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java index 85022d9..00b60d3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java @@ -28,6 +28,7 @@ import static org.mockito.Mockito.spy; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -402,6 +403,7 @@ public void testRMDTSecretManagerStateStore( RMDelegationTokenIdentifier dtId1 = new RMDelegationTokenIdentifier(new Text("owner1"), new Text("renewer1"), new Text("realuser1")); + byte[] tokenBeforeStore = dtId1.getBytes(); Long renewDate1 = new Long(System.currentTimeMillis()); int sequenceNumber = 1111; store.storeRMDelegationTokenAndSequenceNumber(dtId1, renewDate1, @@ -423,6 +425,10 @@ public void testRMDTSecretManagerStateStore( Assert.assertEquals(keySet, secretManagerState.getMasterKeyState()); Assert.assertEquals(sequenceNumber, secretManagerState.getDTSequenceNumber()); + RMDelegationTokenIdentifier tokenAfterStore = + secretManagerState.getTokenState().keySet().iterator().next(); + Assert.assertTrue(Arrays.equals(tokenBeforeStore, + tokenAfterStore.getBytes())); // update RM delegation token; renewDate1 = new Long(System.currentTimeMillis());