diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java index dc56ffa..02f15f1 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java @@ -58,7 +58,7 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; -import org.apache.hadoop.yarn.util.ProtoUtils; +import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.Records; import com.google.common.util.concurrent.ThreadFactoryBuilder; @@ -361,7 +361,7 @@ protected ContainerManagementProtocol getCMProxy(ContainerId containerID, UserGroupInformation.createRemoteUser(containerID.toString()); Token token = - ProtoUtils.convertFromProtoFormat(containerToken, cmAddr); + ConverterUtils.convertFromYarn(containerToken, cmAddr); user.addToken(token); ContainerManagementProtocol proxy = user diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java index a74c8ffa..77a0ad2 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java @@ -75,7 +75,7 @@ import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier; -import org.apache.hadoop.yarn.util.ProtoUtils; +import org.apache.hadoop.yarn.util.ConverterUtils; public class ClientServiceDelegate { private static final Log LOG = LogFactory.getLog(ClientServiceDelegate.class); @@ -183,7 +183,7 @@ private MRClientProtocol getProxy() throws IOException { org.apache.hadoop.yarn.api.records.Token clientToAMToken = application.getClientToAMToken(); Token token = - ProtoUtils.convertFromProtoFormat(clientToAMToken, serviceAddr); + ConverterUtils.convertFromYarn(clientToAMToken, serviceAddr); newUgi.addToken(token); } LOG.debug("Connecting to " + serviceAddr); diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java index 349a019..03649c3 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java @@ -44,7 +44,7 @@ import org.apache.hadoop.yarn.client.YarnClientImpl; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.util.ProtoUtils; +import org.apache.hadoop.yarn.util.ConverterUtils; public class ResourceMgrDelegate extends YarnClientImpl { private static final Log LOG = LogFactory.getLog(ResourceMgrDelegate.class); @@ -111,7 +111,7 @@ InetSocketAddress getConnectAddress() { public Token getDelegationToken(Text renewer) throws IOException, InterruptedException { try { - return ProtoUtils.convertFromProtoFormat( + return ConverterUtils.convertFromYarn( super.getRMDelegationToken(renewer), rmAddress); } catch (YarnException e) { throw new IOException(e); diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java index aefa755..3d2952b 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java @@ -79,12 +79,10 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.security.client.RMDelegationTokenSelector; import org.apache.hadoop.yarn.util.ConverterUtils; -import org.apache.hadoop.yarn.util.ProtoUtils; import com.google.common.annotations.VisibleForTesting; @@ -209,7 +207,7 @@ void addHistoryToken(Credentials ts) throws IOException, InterruptedException { org.apache.hadoop.yarn.api.records.Token mrDelegationToken; mrDelegationToken = hsProxy.getDelegationToken(request) .getDelegationToken(); - return ProtoUtils.convertFromProtoFormat(mrDelegationToken, + return ConverterUtils.convertFromYarn(mrDelegationToken, hsProxy.getConnectAddress()); } diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestJHSSecurity.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestJHSSecurity.java index 2181b6b..d793667 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestJHSSecurity.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestJHSSecurity.java @@ -48,7 +48,7 @@ import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.ipc.YarnRPC; -import org.apache.hadoop.yarn.util.ProtoUtils; +import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.Records; import org.apache.log4j.Level; import org.apache.log4j.LogManager; @@ -262,7 +262,7 @@ public Void run() throws IOException { private MRClientProtocol getMRClientProtocol(Token token, final InetSocketAddress hsAddress, String user, final Configuration conf) { UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user); - ugi.addToken(ProtoUtils.convertFromProtoFormat(token, hsAddress)); + ugi.addToken(ConverterUtils.convertFromYarn(token, hsAddress)); final YarnRPC rpc = YarnRPC.create(conf); MRClientProtocol hsWithDT = ugi diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java index 5d88d51..f38f91e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java @@ -38,6 +38,7 @@ import org.apache.hadoop.yarn.api.records.impl.pb.NMTokenPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.NodeReportPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.PreemptionMessagePBImpl; +import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils; import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl; import org.apache.hadoop.yarn.proto.YarnProtos.ContainerProto; import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStatusProto; @@ -47,7 +48,6 @@ import org.apache.hadoop.yarn.proto.YarnServiceProtos.AllocateResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.AllocateResponseProtoOrBuilder; import org.apache.hadoop.yarn.proto.YarnServiceProtos.NMTokenProto; -import org.apache.hadoop.yarn.util.ProtoUtils; @Private @Unstable diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/FinishApplicationMasterRequestPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/FinishApplicationMasterRequestPBImpl.java index 0c10646..cf52896 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/FinishApplicationMasterRequestPBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/FinishApplicationMasterRequestPBImpl.java @@ -25,11 +25,11 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl; +import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptIdProto; import org.apache.hadoop.yarn.proto.YarnProtos.FinalApplicationStatusProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.FinishApplicationMasterRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.FinishApplicationMasterRequestProtoOrBuilder; -import org.apache.hadoop.yarn.util.ProtoUtils; @Private @Unstable diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java index 1165b2f..e2962e3 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java @@ -30,12 +30,12 @@ import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils; import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationACLMapProto; import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.RegisterApplicationMasterResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.RegisterApplicationMasterResponseProtoOrBuilder; -import org.apache.hadoop.yarn.util.ProtoUtils; import com.google.protobuf.ByteString; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StartContainerResponsePBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StartContainerResponsePBImpl.java index 77bbb99..dba83ce 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StartContainerResponsePBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StartContainerResponsePBImpl.java @@ -28,10 +28,10 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse; +import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils; import org.apache.hadoop.yarn.proto.YarnProtos.StringBytesMapProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.StartContainerResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.StartContainerResponseProtoOrBuilder; -import org.apache.hadoop.yarn.util.ProtoUtils; import com.google.protobuf.ByteString; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java index 5db4d0e..b08db0d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java @@ -35,7 +35,6 @@ import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationResourceUsageReportProto; import org.apache.hadoop.yarn.proto.YarnProtos.FinalApplicationStatusProto; import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationStateProto; -import org.apache.hadoop.yarn.util.ProtoUtils; @Private @Unstable diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java index 9734c3a..02e0d3b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java @@ -37,7 +37,6 @@ import org.apache.hadoop.yarn.proto.YarnProtos.StringBytesMapProto; import org.apache.hadoop.yarn.proto.YarnProtos.StringLocalResourceMapProto; import org.apache.hadoop.yarn.proto.YarnProtos.StringStringMapProto; -import org.apache.hadoop.yarn.util.ProtoUtils; import com.google.protobuf.ByteString; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerStatusPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerStatusPBImpl.java index b8558c6..a52ae4c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerStatusPBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerStatusPBImpl.java @@ -28,7 +28,6 @@ import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStateProto; import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStatusProto; import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStatusProtoOrBuilder; -import org.apache.hadoop.yarn.util.ProtoUtils; @Private @Unstable diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/LocalResourcePBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/LocalResourcePBImpl.java index 2d0929a..d2caf0b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/LocalResourcePBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/LocalResourcePBImpl.java @@ -29,7 +29,6 @@ import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceTypeProto; import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceVisibilityProto; import org.apache.hadoop.yarn.proto.YarnProtos.URLProto; -import org.apache.hadoop.yarn.util.ProtoUtils; @Private @Unstable diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeReportPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeReportPBImpl.java index 06d4424..e1a66d7 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeReportPBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeReportPBImpl.java @@ -28,7 +28,6 @@ import org.apache.hadoop.yarn.proto.YarnProtos.NodeReportProto; import org.apache.hadoop.yarn.proto.YarnProtos.NodeReportProtoOrBuilder; import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto; -import org.apache.hadoop.yarn.util.ProtoUtils; @Private @Unstable diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoBase.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoBase.java index 0df75a2..4e8c39b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoBase.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoBase.java @@ -22,7 +22,6 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.yarn.util.ProtoUtils; import com.google.protobuf.ByteString; import com.google.protobuf.Message; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java new file mode 100644 index 0000000..b660f7d --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java @@ -0,0 +1,209 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records.impl.pb; + +import java.nio.ByteBuffer; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.records.AMCommand; +import org.apache.hadoop.yarn.api.records.ApplicationAccessType; +import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; +import org.apache.hadoop.yarn.api.records.ContainerState; +import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.LocalResourceType; +import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; +import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.NodeState; +import org.apache.hadoop.yarn.api.records.QueueACL; +import org.apache.hadoop.yarn.api.records.QueueState; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.proto.YarnProtos.AMCommandProto; +import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAccessTypeProto; +import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationResourceUsageReportProto; +import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStateProto; +import org.apache.hadoop.yarn.proto.YarnProtos.FinalApplicationStatusProto; +import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceTypeProto; +import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceVisibilityProto; +import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto; +import org.apache.hadoop.yarn.proto.YarnProtos.NodeStateProto; +import org.apache.hadoop.yarn.proto.YarnProtos.QueueACLProto; +import org.apache.hadoop.yarn.proto.YarnProtos.QueueStateProto; +import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationStateProto; + +import com.google.protobuf.ByteString; + +@Private +@Unstable +public class ProtoUtils { + + + /* + * ContainerState + */ + private static String CONTAINER_STATE_PREFIX = "C_"; + public static ContainerStateProto convertToProtoFormat(ContainerState e) { + return ContainerStateProto.valueOf(CONTAINER_STATE_PREFIX + e.name()); + } + public static ContainerState convertFromProtoFormat(ContainerStateProto e) { + return ContainerState.valueOf(e.name().replace(CONTAINER_STATE_PREFIX, "")); + } + + /* + * NodeState + */ + private static String NODE_STATE_PREFIX = "NS_"; + public static NodeStateProto convertToProtoFormat(NodeState e) { + return NodeStateProto.valueOf(NODE_STATE_PREFIX + e.name()); + } + public static NodeState convertFromProtoFormat(NodeStateProto e) { + return NodeState.valueOf(e.name().replace(NODE_STATE_PREFIX, "")); + } + + /* + * NodeId + */ + public static NodeIdProto convertToProtoFormat(NodeId e) { + return ((NodeIdPBImpl)e).getProto(); + } + public static NodeId convertFromProtoFormat(NodeIdProto e) { + return new NodeIdPBImpl(e); + } + + /* + * YarnApplicationState + */ + public static YarnApplicationStateProto convertToProtoFormat(YarnApplicationState e) { + return YarnApplicationStateProto.valueOf(e.name()); + } + public static YarnApplicationState convertFromProtoFormat(YarnApplicationStateProto e) { + return YarnApplicationState.valueOf(e.name()); + } + + /* + * ApplicationResourceUsageReport + */ + public static ApplicationResourceUsageReportProto convertToProtoFormat(ApplicationResourceUsageReport e) { + return ((ApplicationResourceUsageReportPBImpl)e).getProto(); + } + + public static ApplicationResourceUsageReport convertFromProtoFormat(ApplicationResourceUsageReportProto e) { + return new ApplicationResourceUsageReportPBImpl(e); + } + + /* + * FinalApplicationStatus + */ + private static String FINAL_APPLICATION_STATUS_PREFIX = "APP_"; + public static FinalApplicationStatusProto convertToProtoFormat(FinalApplicationStatus e) { + return FinalApplicationStatusProto.valueOf(FINAL_APPLICATION_STATUS_PREFIX + e.name()); + } + public static FinalApplicationStatus convertFromProtoFormat(FinalApplicationStatusProto e) { + return FinalApplicationStatus.valueOf(e.name().replace(FINAL_APPLICATION_STATUS_PREFIX, "")); + } + + /* + * LocalResourceType + */ + public static LocalResourceTypeProto convertToProtoFormat(LocalResourceType e) { + return LocalResourceTypeProto.valueOf(e.name()); + } + public static LocalResourceType convertFromProtoFormat(LocalResourceTypeProto e) { + return LocalResourceType.valueOf(e.name()); + } + + /* + * LocalResourceVisibility + */ + public static LocalResourceVisibilityProto convertToProtoFormat(LocalResourceVisibility e) { + return LocalResourceVisibilityProto.valueOf(e.name()); + } + public static LocalResourceVisibility convertFromProtoFormat(LocalResourceVisibilityProto e) { + return LocalResourceVisibility.valueOf(e.name()); + } + + /* + * AMCommand + */ + public static AMCommandProto convertToProtoFormat(AMCommand e) { + return AMCommandProto.valueOf(e.name()); + } + public static AMCommand convertFromProtoFormat(AMCommandProto e) { + return AMCommand.valueOf(e.name()); + } + + /* + * ByteBuffer + */ + public static ByteBuffer convertFromProtoFormat(ByteString byteString) { + int capacity = byteString.asReadOnlyByteBuffer().rewind().remaining(); + byte[] b = new byte[capacity]; + byteString.asReadOnlyByteBuffer().get(b, 0, capacity); + return ByteBuffer.wrap(b); + } + + public static ByteString convertToProtoFormat(ByteBuffer byteBuffer) { +// return ByteString.copyFrom((ByteBuffer)byteBuffer.duplicate().rewind()); + int oldPos = byteBuffer.position(); + byteBuffer.rewind(); + ByteString bs = ByteString.copyFrom(byteBuffer); + byteBuffer.position(oldPos); + return bs; + } + + /* + * QueueState + */ + private static String QUEUE_STATE_PREFIX = "Q_"; + public static QueueStateProto convertToProtoFormat(QueueState e) { + return QueueStateProto.valueOf(QUEUE_STATE_PREFIX + e.name()); + } + public static QueueState convertFromProtoFormat(QueueStateProto e) { + return QueueState.valueOf(e.name().replace(QUEUE_STATE_PREFIX, "")); + } + + /* + * QueueACL + */ + private static String QUEUE_ACL_PREFIX = "QACL_"; + public static QueueACLProto convertToProtoFormat(QueueACL e) { + return QueueACLProto.valueOf(QUEUE_ACL_PREFIX + e.name()); + } + public static QueueACL convertFromProtoFormat(QueueACLProto e) { + return QueueACL.valueOf(e.name().replace(QUEUE_ACL_PREFIX, "")); + } + + + /* + * ApplicationAccessType + */ + private static String APP_ACCESS_TYPE_PREFIX = "APPACCESS_"; + + public static ApplicationAccessTypeProto convertToProtoFormat( + ApplicationAccessType e) { + return ApplicationAccessTypeProto.valueOf(APP_ACCESS_TYPE_PREFIX + + e.name()); + } + + public static ApplicationAccessType convertFromProtoFormat( + ApplicationAccessTypeProto e) { + return ApplicationAccessType.valueOf(e.name().replace( + APP_ACCESS_TYPE_PREFIX, "")); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueInfoPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueInfoPBImpl.java index b20cf44..529bd8b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueInfoPBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueInfoPBImpl.java @@ -31,7 +31,6 @@ import org.apache.hadoop.yarn.proto.YarnProtos.QueueInfoProto; import org.apache.hadoop.yarn.proto.YarnProtos.QueueInfoProtoOrBuilder; import org.apache.hadoop.yarn.proto.YarnProtos.QueueStateProto; -import org.apache.hadoop.yarn.util.ProtoUtils; @Private @Unstable diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueUserACLInfoPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueUserACLInfoPBImpl.java index ba0da93..cf48410 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueUserACLInfoPBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueUserACLInfoPBImpl.java @@ -29,7 +29,6 @@ import org.apache.hadoop.yarn.proto.YarnProtos.QueueACLProto; import org.apache.hadoop.yarn.proto.YarnProtos.QueueUserACLInfoProto; import org.apache.hadoop.yarn.proto.YarnProtos.QueueUserACLInfoProtoOrBuilder; -import org.apache.hadoop.yarn.util.ProtoUtils; @Private @Unstable diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/TokenPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/TokenPBImpl.java index 2ad25be..2835cbb 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/TokenPBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/TokenPBImpl.java @@ -25,7 +25,6 @@ import org.apache.hadoop.security.proto.SecurityProtos.TokenProto; import org.apache.hadoop.security.proto.SecurityProtos.TokenProtoOrBuilder; import org.apache.hadoop.yarn.api.records.Token; -import org.apache.hadoop.yarn.util.ProtoUtils; import com.google.protobuf.ByteString; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/ProtoUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/ProtoUtils.java deleted file mode 100644 index b283bac..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/ProtoUtils.java +++ /dev/null @@ -1,236 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.util; - -import java.net.InetSocketAddress; -import java.nio.ByteBuffer; - -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.security.SecurityUtil; -import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.security.token.TokenIdentifier; -import org.apache.hadoop.yarn.api.records.AMCommand; -import org.apache.hadoop.yarn.api.records.ApplicationAccessType; -import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; -import org.apache.hadoop.yarn.api.records.ContainerState; -import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; -import org.apache.hadoop.yarn.api.records.LocalResourceType; -import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; -import org.apache.hadoop.yarn.api.records.NodeId; -import org.apache.hadoop.yarn.api.records.NodeState; -import org.apache.hadoop.yarn.api.records.QueueACL; -import org.apache.hadoop.yarn.api.records.QueueState; -import org.apache.hadoop.yarn.api.records.YarnApplicationState; -import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationResourceUsageReportPBImpl; -import org.apache.hadoop.yarn.api.records.impl.pb.NodeIdPBImpl; -import org.apache.hadoop.yarn.proto.YarnProtos.AMCommandProto; -import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAccessTypeProto; -import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationResourceUsageReportProto; -import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStateProto; -import org.apache.hadoop.yarn.proto.YarnProtos.FinalApplicationStatusProto; -import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceTypeProto; -import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceVisibilityProto; -import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto; -import org.apache.hadoop.yarn.proto.YarnProtos.NodeStateProto; -import org.apache.hadoop.yarn.proto.YarnProtos.QueueACLProto; -import org.apache.hadoop.yarn.proto.YarnProtos.QueueStateProto; -import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationStateProto; - -import com.google.protobuf.ByteString; - -@Private -@Unstable -public class ProtoUtils { - - - /* - * ContainerState - */ - private static String CONTAINER_STATE_PREFIX = "C_"; - public static ContainerStateProto convertToProtoFormat(ContainerState e) { - return ContainerStateProto.valueOf(CONTAINER_STATE_PREFIX + e.name()); - } - public static ContainerState convertFromProtoFormat(ContainerStateProto e) { - return ContainerState.valueOf(e.name().replace(CONTAINER_STATE_PREFIX, "")); - } - - /* - * NodeState - */ - private static String NODE_STATE_PREFIX = "NS_"; - public static NodeStateProto convertToProtoFormat(NodeState e) { - return NodeStateProto.valueOf(NODE_STATE_PREFIX + e.name()); - } - public static NodeState convertFromProtoFormat(NodeStateProto e) { - return NodeState.valueOf(e.name().replace(NODE_STATE_PREFIX, "")); - } - - /* - * NodeId - */ - public static NodeIdProto convertToProtoFormat(NodeId e) { - return ((NodeIdPBImpl)e).getProto(); - } - public static NodeId convertFromProtoFormat(NodeIdProto e) { - return new NodeIdPBImpl(e); - } - - /* - * YarnApplicationState - */ - public static YarnApplicationStateProto convertToProtoFormat(YarnApplicationState e) { - return YarnApplicationStateProto.valueOf(e.name()); - } - public static YarnApplicationState convertFromProtoFormat(YarnApplicationStateProto e) { - return YarnApplicationState.valueOf(e.name()); - } - - /* - * ApplicationResourceUsageReport - */ - public static ApplicationResourceUsageReportProto convertToProtoFormat(ApplicationResourceUsageReport e) { - return ((ApplicationResourceUsageReportPBImpl)e).getProto(); - } - - public static ApplicationResourceUsageReport convertFromProtoFormat(ApplicationResourceUsageReportProto e) { - return new ApplicationResourceUsageReportPBImpl(e); - } - - /* - * FinalApplicationStatus - */ - private static String FINAL_APPLICATION_STATUS_PREFIX = "APP_"; - public static FinalApplicationStatusProto convertToProtoFormat(FinalApplicationStatus e) { - return FinalApplicationStatusProto.valueOf(FINAL_APPLICATION_STATUS_PREFIX + e.name()); - } - public static FinalApplicationStatus convertFromProtoFormat(FinalApplicationStatusProto e) { - return FinalApplicationStatus.valueOf(e.name().replace(FINAL_APPLICATION_STATUS_PREFIX, "")); - } - - /* - * LocalResourceType - */ - public static LocalResourceTypeProto convertToProtoFormat(LocalResourceType e) { - return LocalResourceTypeProto.valueOf(e.name()); - } - public static LocalResourceType convertFromProtoFormat(LocalResourceTypeProto e) { - return LocalResourceType.valueOf(e.name()); - } - - /* - * LocalResourceVisibility - */ - public static LocalResourceVisibilityProto convertToProtoFormat(LocalResourceVisibility e) { - return LocalResourceVisibilityProto.valueOf(e.name()); - } - public static LocalResourceVisibility convertFromProtoFormat(LocalResourceVisibilityProto e) { - return LocalResourceVisibility.valueOf(e.name()); - } - - /* - * AMCommand - */ - public static AMCommandProto convertToProtoFormat(AMCommand e) { - return AMCommandProto.valueOf(e.name()); - } - public static AMCommand convertFromProtoFormat(AMCommandProto e) { - return AMCommand.valueOf(e.name()); - } - - /* - * ByteBuffer - */ - public static ByteBuffer convertFromProtoFormat(ByteString byteString) { - int capacity = byteString.asReadOnlyByteBuffer().rewind().remaining(); - byte[] b = new byte[capacity]; - byteString.asReadOnlyByteBuffer().get(b, 0, capacity); - return ByteBuffer.wrap(b); - } - - public static ByteString convertToProtoFormat(ByteBuffer byteBuffer) { -// return ByteString.copyFrom((ByteBuffer)byteBuffer.duplicate().rewind()); - int oldPos = byteBuffer.position(); - byteBuffer.rewind(); - ByteString bs = ByteString.copyFrom(byteBuffer); - byteBuffer.position(oldPos); - return bs; - } - - /* - * QueueState - */ - private static String QUEUE_STATE_PREFIX = "Q_"; - public static QueueStateProto convertToProtoFormat(QueueState e) { - return QueueStateProto.valueOf(QUEUE_STATE_PREFIX + e.name()); - } - public static QueueState convertFromProtoFormat(QueueStateProto e) { - return QueueState.valueOf(e.name().replace(QUEUE_STATE_PREFIX, "")); - } - - /* - * QueueACL - */ - private static String QUEUE_ACL_PREFIX = "QACL_"; - public static QueueACLProto convertToProtoFormat(QueueACL e) { - return QueueACLProto.valueOf(QUEUE_ACL_PREFIX + e.name()); - } - public static QueueACL convertFromProtoFormat(QueueACLProto e) { - return QueueACL.valueOf(e.name().replace(QUEUE_ACL_PREFIX, "")); - } - - - /* - * ApplicationAccessType - */ - private static String APP_ACCESS_TYPE_PREFIX = "APPACCESS_"; - - public static ApplicationAccessTypeProto convertToProtoFormat( - ApplicationAccessType e) { - return ApplicationAccessTypeProto.valueOf(APP_ACCESS_TYPE_PREFIX - + e.name()); - } - - public static ApplicationAccessType convertFromProtoFormat( - ApplicationAccessTypeProto e) { - return ApplicationAccessType.valueOf(e.name().replace( - APP_ACCESS_TYPE_PREFIX, "")); - } - - /** - * Convert a protobuf token into a rpc token and set its service - * - * @param protoToken the yarn token - * @param serviceAddr the connect address for the service - * @return rpc token - */ - public static Token convertFromProtoFormat( - org.apache.hadoop.yarn.api.records.Token protoToken, - InetSocketAddress serviceAddr) { - Token token = new Token(protoToken.getIdentifier().array(), - protoToken.getPassword().array(), - new Text(protoToken.getKind()), - new Text(protoToken.getService())); - if (serviceAddr != null) { - SecurityUtil.setTokenService(token, serviceAddr); - } - return token; - } -} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/NMClientImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/NMClientImpl.java index 042af51..3c99df1 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/NMClientImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/NMClientImpl.java @@ -49,7 +49,7 @@ import org.apache.hadoop.yarn.ipc.RPCUtil; import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; -import org.apache.hadoop.yarn.util.ProtoUtils; +import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.Records; /** @@ -182,7 +182,7 @@ protected void serviceStart() throws Exception { UserGroupInformation.createRemoteUser(containerId.toString()); org.apache.hadoop.security.token.Token token = - ProtoUtils.convertFromProtoFormat(containerToken, containerAddress); + ConverterUtils.convertFromYarn(containerToken, containerAddress); currentUser.addToken(token); containerManager = currentUser diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java index d350003..bb93b91 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java @@ -20,6 +20,7 @@ import static org.apache.hadoop.yarn.util.StringHelper._split; +import java.net.InetSocketAddress; import java.net.URI; import java.net.URISyntaxException; import java.util.HashMap; @@ -29,6 +30,10 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -202,4 +207,24 @@ public static ApplicationId toApplicationId( + appIdStr, n); } } + + /** + * Convert a protobuf token into a rpc token and set its service + * + * @param protoToken the yarn token + * @param serviceAddr the connect address for the service + * @return rpc token + */ + public static Token convertFromYarn( + org.apache.hadoop.yarn.api.records.Token protoToken, + InetSocketAddress serviceAddr) { + Token token = new Token(protoToken.getIdentifier().array(), + protoToken.getPassword().array(), + new Text(protoToken.getKind()), + new Text(protoToken.getService())); + if (serviceAddr != null) { + SecurityUtil.setTokenService(token, serviceAddr); + } + return token; + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java index 5b7126b..673c7e4 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java @@ -24,9 +24,6 @@ import java.security.PrivilegedAction; import java.util.Map; -import javax.crypto.SecretKey; - -import org.apache.commons.codec.binary.Base64; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -59,7 +56,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptLaunchFailedEvent; -import org.apache.hadoop.yarn.util.ProtoUtils; +import org.apache.hadoop.yarn.util.ConverterUtils; /** * The launch of the AM itself. @@ -138,7 +135,7 @@ protected ContainerManagementProtocol getContainerMgrProxy( .createRemoteUser(containerId.toString()); if (UserGroupInformation.isSecurityEnabled()) { Token token = - ProtoUtils.convertFromProtoFormat(masterContainer + ConverterUtils.convertFromYarn(masterContainer .getContainerToken(), containerManagerBindAddress); currentUser.addToken(token); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMTokens.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMTokens.java index e1b7f08..68caa9b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMTokens.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMTokens.java @@ -66,7 +66,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.security.RMDelegationTokenSecretManager; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.server.utils.BuilderUtils; -import org.apache.hadoop.yarn.util.ProtoUtils; +import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.Records; import org.junit.Before; import org.junit.Test; @@ -409,7 +409,7 @@ private ApplicationClientProtocol getClientRMProtocolWithDT( UserGroupInformation ugi = UserGroupInformation .createRemoteUser(user); - ugi.addToken(ProtoUtils.convertFromProtoFormat(token, rmAddress)); + ugi.addToken(ConverterUtils.convertFromYarn(token, rmAddress)); final YarnRPC rpc = YarnRPC.create(conf); ApplicationClientProtocol clientRMWithDT = ugi diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java index 019aec1..e667633 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java @@ -65,7 +65,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer; import org.apache.hadoop.yarn.server.utils.BuilderUtils; -import org.apache.hadoop.yarn.util.ProtoUtils; +import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.log4j.Level; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; @@ -600,7 +600,7 @@ public void testRMDelegationTokenRestoredOnRMRestart() throws Exception { org.apache.hadoop.yarn.api.records.Token delegationToken1 = response1.getRMDelegationToken(); Token token1 = - ProtoUtils.convertFromProtoFormat(delegationToken1, null); + ConverterUtils.convertFromYarn(delegationToken1, null); RMDelegationTokenIdentifier dtId1 = token1.decodeIdentifier(); HashSet tokenIdentSet = @@ -639,7 +639,7 @@ public void testRMDelegationTokenRestoredOnRMRestart() throws Exception { org.apache.hadoop.yarn.api.records.Token delegationToken2 = response2.getRMDelegationToken(); Token token2 = - ProtoUtils.convertFromProtoFormat(delegationToken2, null); + ConverterUtils.convertFromYarn(delegationToken2, null); RMDelegationTokenIdentifier dtId2 = token2.decodeIdentifier(); // cancel token2 diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java index 5148095..0492471 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java @@ -66,7 +66,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockRMWithCustomAMLauncher; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.utils.BuilderUtils; -import org.apache.hadoop.yarn.util.ProtoUtils; +import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.Records; import org.junit.Test; @@ -255,7 +255,7 @@ public RegisterApplicationMasterResponse run() { // Verify denial for a malicious user UserGroupInformation ugi = UserGroupInformation.createRemoteUser("me"); Token token = - ProtoUtils.convertFromProtoFormat(clientToAMToken, am.address); + ConverterUtils.convertFromYarn(clientToAMToken, am.address); // Malicious user, messes with appId ClientToAMTokenIdentifier maliciousID = diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestRMDelegationTokens.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestRMDelegationTokens.java index 684320d..6cc0a18 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestRMDelegationTokens.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestRMDelegationTokens.java @@ -42,7 +42,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; -import org.apache.hadoop.yarn.util.ProtoUtils; +import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.log4j.Level; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; @@ -101,7 +101,7 @@ public void testRMDTMasterKeyStateOnRollingMasterKey() throws Exception { org.apache.hadoop.yarn.api.records.Token delegationToken = response.getRMDelegationToken(); Token token1 = - ProtoUtils.convertFromProtoFormat(delegationToken, null); + ConverterUtils.convertFromYarn(delegationToken, null); RMDelegationTokenIdentifier dtId1 = token1.decodeIdentifier(); // wait for the first rollMasterKey