From 74689b17b5ca7b85bca9cd717e91c9836f403d8a Mon Sep 17 00:00:00 2001 From: Sunil G Date: Tue, 1 Aug 2017 20:25:10 +0530 Subject: [PATCH] YARN-6789 --- .../apache/hadoop/mapred/ResourceMgrDelegate.java | 7 + .../apache/hadoop/mapred/TestClientRedirect.java | 9 + .../hadoop/yarn/api/ApplicationClientProtocol.java | 19 ++- .../GetAllResourceTypeInfoRequest.java | 35 ++++ .../GetAllResourceTypeInfoResponse.java | 60 +++++++ .../hadoop/yarn/api/records/ResourceTypeInfo.java | 174 ++++++++++++++++++++ .../main/proto/applicationclient_protocol.proto | 1 + .../src/main/proto/yarn_protos.proto | 6 + .../src/main/proto/yarn_service_protos.proto | 7 + .../apache/hadoop/yarn/client/api/YarnClient.java | 19 +++ .../yarn/client/api/impl/YarnClientImpl.java | 14 +- .../ApplicationClientProtocolPBClientImpl.java | 18 ++ .../ApplicationClientProtocolPBServiceImpl.java | 21 +++ .../pb/GetAllResourceTypeInfoRequestPBImpl.java | 55 +++++++ .../pb/GetAllResourceTypeInfoResponsePBImpl.java | 181 +++++++++++++++++++++ .../records/impl/pb/ResourceTypeInfoPBImpl.java | 92 +++++++++++ .../amrmproxy/MockResourceManagerFacade.java | 8 + .../server/resourcemanager/ClientRMService.java | 19 +++ .../resource/ResourceProfilesManager.java | 8 + .../resource/ResourceProfilesManagerImpl.java | 39 +++++ 20 files changed, 789 insertions(+), 3 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAllResourceTypeInfoRequest.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAllResourceTypeInfoResponse.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceTypeInfo.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllResourceTypeInfoRequestPBImpl.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllResourceTypeInfoResponsePBImpl.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceTypeInfoPBImpl.java diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java index a365f80..94f741a 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java @@ -67,6 +67,7 @@ import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceTypeInfo; import org.apache.hadoop.yarn.api.records.SignalContainerCommand; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.api.records.YarnClusterMetrics; @@ -530,4 +531,10 @@ public Resource getResourceProfile(String profile) throws YarnException, IOException { return client.getResourceProfile(profile); } + + @Override + public List getResourceTypeInfo() + throws YarnException, IOException { + return client.getResourceTypeInfo(); + } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java index cc50be0..a9b4626 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java @@ -74,6 +74,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsRequest; @@ -511,6 +513,13 @@ public GetResourceProfileResponse getResourceProfile( GetResourceProfileRequest request) throws YarnException, IOException { return null; } + + @Override + public GetAllResourceTypeInfoResponse getResourceTypeInfo( + GetAllResourceTypeInfoRequest request) + throws YarnException, IOException { + return null; + } } class HistoryService extends AMService implements HSClientProtocol { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java index 09d0f43..808bffa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java @@ -67,6 +67,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileResponse; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; @@ -79,6 +81,8 @@ import org.apache.hadoop.yarn.api.records.YarnClusterMetrics; import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllResourceTypeInfoRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllResourceTypeInfoResponseProto; /** *

The protocol between clients and the ResourceManager @@ -625,5 +629,18 @@ GetAllResourceProfilesResponse getResourceProfiles( GetResourceProfileResponse getResourceProfile( GetResourceProfileRequest request) throws YarnException, IOException; - + /** + *

+ * The interface to get the details for a specific resource profile. + *

+ * @param request request to get the details of a resource profile + * @return Response containing the details for a particular resource profile + * @throws YarnException if resource profiles are not enabled on the RM or + * the profile cannot be found + * @throws IOException in case of other errors + */ + @Public + @Unstable + GetAllResourceTypeInfoResponse getResourceTypeInfo( + GetAllResourceTypeInfoRequest request) throws YarnException, IOException; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAllResourceTypeInfoRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAllResourceTypeInfoRequest.java new file mode 100644 index 0000000..3bda4f5 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAllResourceTypeInfoRequest.java @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.util.Records; + +/** + * Request class for getting all the resource profiles from the RM. + */ +@InterfaceAudience.Public +@InterfaceStability.Unstable +public abstract class GetAllResourceTypeInfoRequest { + + public static GetAllResourceTypeInfoRequest newInstance() { + return Records.newRecord(GetAllResourceTypeInfoRequest.class); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAllResourceTypeInfoResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAllResourceTypeInfoResponse.java new file mode 100644 index 0000000..b57b96d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAllResourceTypeInfoResponse.java @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.api.records.ResourceTypeInfo; +import org.apache.hadoop.yarn.util.Records; + +import java.util.List; + +/** + * Response class for getting all the resource profiles from the RM. + */ +@InterfaceAudience.Public +@InterfaceStability.Unstable +public abstract class GetAllResourceTypeInfoResponse { + + public static GetAllResourceTypeInfoResponse newInstance() { + return Records.newRecord(GetAllResourceTypeInfoResponse.class); + } + + public abstract void setResourceTypeInfo(List resourceTypes); + + public abstract List getResourceTypeInfo(); + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (other == null || !(other instanceof GetAllResourceTypeInfoResponse)) { + return false; + } + return ((GetAllResourceTypeInfoResponse) other).getResourceTypeInfo() + .equals(this.getResourceTypeInfo()); + } + + @Override + public int hashCode() { + return this.getResourceTypeInfo().hashCode(); + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceTypeInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceTypeInfo.java new file mode 100644 index 0000000..5b71ee7 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceTypeInfo.java @@ -0,0 +1,174 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Evolving; +import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes; +import org.apache.hadoop.yarn.util.Records; + +/** + * Class to encapsulate information about a ResourceType - the name of the + * resource, the units(milli, micro, etc), the type(countable). + */ +public abstract class ResourceTypeInfo implements Comparable { + + /** + * Get the name for the resource. + * + * @return resource name + */ + public abstract String getName(); + + /** + * Set the name for the resource. + * + * @param rName + * name for the resource + */ + public abstract void setName(String rName); + + /** + * Get units for the resource. + * + * @return units for the resource + */ + public abstract String getMinimumUnit(); + + /** + * Set the units for the resource. + * + * @param rUnits + * units for the resource + */ + public abstract void setMinimumUnit(String rUnits); + + /** + * Get the resource type. + * + * @return the resource type + */ + public abstract ResourceTypes getResourceType(); + + /** + * Set the resource type. + * + * @param type + * the resource type + */ + public abstract void setResourceType(ResourceTypes type); + + /** + * Create a new instance of ResourceInformation from another object. + * + * @param other + * the object from which the new object should be created + * @return the new ResourceInformation object + */ + @Public + @Evolving + public static ResourceTypeInfo newInstance(ResourceTypeInfo other) { + ResourceTypeInfo resourceType = Records.newRecord(ResourceTypeInfo.class); + copy(other, resourceType); + return resourceType; + } + + @Public + @Evolving + public static ResourceTypeInfo newInstance(String name, String units, + ResourceTypes type) { + ResourceTypeInfo resourceType = Records.newRecord(ResourceTypeInfo.class); + resourceType.setName(name); + resourceType.setResourceType(type); + resourceType.setMinimumUnit(units); + return resourceType; + } + + @Public + @Evolving + public static ResourceTypeInfo newInstance(String name, String units) { + return ResourceTypeInfo.newInstance(name, units, ResourceTypes.COUNTABLE); + } + + @Public + @Evolving + public static ResourceTypeInfo newInstance(String name) { + return ResourceTypeInfo.newInstance(name, ""); + } + + /** + * Copies the content of the source ResourceInformation object to the + * destination object, overwriting all properties of the destination object. + * + * @param src + * Source ResourceInformation object + * @param dst + * Destination ResourceInformation object + */ + + public static void copy(ResourceTypeInfo src, ResourceTypeInfo dst) { + dst.setName(src.getName()); + dst.setResourceType(src.getResourceType()); + dst.setMinimumUnit(src.getMinimumUnit()); + } + + @Override + public String toString() { + return "name: " + this.getName() + ", units: " + this.getMinimumUnit() + + ", type: " + getResourceType(); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (!(obj instanceof ResourceTypeInfo)) { + return false; + } + ResourceTypeInfo r = (ResourceTypeInfo) obj; + return this.getName().equals(r.getName()) + && this.getResourceType().equals(r.getResourceType()) + && this.getMinimumUnit().equals(r.getMinimumUnit()); + } + + @Override + public int hashCode() { + final int prime = 263167; + int result = 939769357 + getName().hashCode(); // prime * result = 939769357 + // initially + result = prime * result + getResourceType().hashCode(); + return result; + } + + @Override + public int compareTo(ResourceTypeInfo other) { + int diff = this.getName().compareTo(other.getName()); + if (diff == 0) { + diff = this.getMinimumUnit().compareTo(other.getMinimumUnit()); + if (diff == 0) { + diff = this.getResourceType().compareTo(other.getResourceType()); + } + } + return diff; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/applicationclient_protocol.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/applicationclient_protocol.proto index aac6732..81adef1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/applicationclient_protocol.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/applicationclient_protocol.proto @@ -63,4 +63,5 @@ service ApplicationClientProtocolService { rpc updateApplicationTimeouts (UpdateApplicationTimeoutsRequestProto) returns (UpdateApplicationTimeoutsResponseProto); rpc getResourceProfiles(GetAllResourceProfilesRequestProto) returns (GetAllResourceProfilesResponseProto); rpc getResourceProfile(GetResourceProfileRequestProto) returns (GetResourceProfileResponseProto); + rpc getResourceTypeInfo(GetAllResourceTypeInfoRequestProto) returns (GetAllResourceTypeInfoResponseProto); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 292cdc7..b95affa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -64,6 +64,12 @@ message ResourceInformationProto { optional ResourceTypesProto type = 4; } +message ResourceTypeInfoProto { + required string name = 1; + optional string units = 2; + optional ResourceTypesProto type = 3; +} + message ResourceProto { optional int64 memory = 1; optional int32 virtual_cores = 2; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto index 6b29806..d69a6ca 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto @@ -295,6 +295,13 @@ message GetResourceProfileResponseProto { required ResourceProto resources = 1; } +message GetAllResourceTypeInfoRequestProto { +} + +message GetAllResourceTypeInfoResponseProto { + repeated ResourceTypeInfoProto resource_type_info = 1; +} + ////////////////////////////////////////////////////// /////// client_NM_Protocol /////////////////////////// ////////////////////////////////////////////////////// diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java index 24d2887..4f5b5bc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java @@ -62,6 +62,7 @@ import org.apache.hadoop.yarn.api.records.ReservationDefinition; import org.apache.hadoop.yarn.api.records.ReservationId; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceTypeInfo; import org.apache.hadoop.yarn.api.records.SignalContainerCommand; import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.YarnApplicationState; @@ -865,6 +866,8 @@ public UpdateApplicationTimeoutsResponse updateApplicationTimeouts( * @throws YarnException if resource profiles are not enabled * @throws IOException in case of other errors */ + @Public + @Unstable public abstract Map getResourceProfiles() throws YarnException, IOException; @@ -878,6 +881,22 @@ public UpdateApplicationTimeoutsResponse updateApplicationTimeouts( * cannot be found * @throws IOException in case of other others */ + @Public + @Unstable public abstract Resource getResourceProfile(String profile) throws YarnException, IOException; + + /** + *

+ * Get available resource types supported by RM. + *

+ * @return list of supported resource types with detailed information + * @throws YarnException if resource profiles are not enabled or the profile + * cannot be found + * @throws IOException in case of other others + */ + @Public + @Unstable + public abstract List getResourceTypeInfo() + throws YarnException, IOException; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java index b22a693..3c30023 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java @@ -44,6 +44,7 @@ import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsRequest; @@ -104,6 +105,7 @@ import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceTypeInfo; import org.apache.hadoop.yarn.api.records.SignalContainerCommand; import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.YarnApplicationState; @@ -952,8 +954,16 @@ public UpdateApplicationTimeoutsResponse updateApplicationTimeouts( @Override public Resource getResourceProfile(String profile) throws YarnException, IOException { - GetResourceProfileRequest request = - GetResourceProfileRequest.newInstance(profile); + GetResourceProfileRequest request = GetResourceProfileRequest + .newInstance(profile); return rmClient.getResourceProfile(request).getResource(); } + + @Override + public List getResourceTypeInfo() + throws YarnException, IOException { + GetAllResourceTypeInfoRequest request = + GetAllResourceTypeInfoRequest.newInstance(); + return rmClient.getResourceTypeInfo(request).getResourceTypeInfo(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ApplicationClientProtocolPBClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ApplicationClientProtocolPBClientImpl.java index 4bad60d..fd5096a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ApplicationClientProtocolPBClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ApplicationClientProtocolPBClientImpl.java @@ -91,6 +91,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileResponse; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.CancelDelegationTokenRequestPBImpl; @@ -153,6 +155,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.SubmitApplicationResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetAllResourceProfilesRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetAllResourceProfilesResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetAllResourceTypeInfoRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetAllResourceTypeInfoResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetResourceProfileRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetResourceProfileResponsePBImpl; import org.apache.hadoop.yarn.exceptions.YarnException; @@ -655,4 +659,18 @@ public GetResourceProfileResponse getResourceProfile( return null; } } + + @Override + public GetAllResourceTypeInfoResponse getResourceTypeInfo( + GetAllResourceTypeInfoRequest request) throws YarnException, IOException { + YarnServiceProtos.GetAllResourceTypeInfoRequestProto requestProto = + ((GetAllResourceTypeInfoRequestPBImpl) request).getProto(); + try { + return new GetAllResourceTypeInfoResponsePBImpl( + proxy.getResourceTypeInfo(null, requestProto)); + } catch (ServiceException e) { + RPCUtil.unwrapAndThrowException(e); + return null; + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ApplicationClientProtocolPBServiceImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ApplicationClientProtocolPBServiceImpl.java index 1c85030..423287e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ApplicationClientProtocolPBServiceImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ApplicationClientProtocolPBServiceImpl.java @@ -59,6 +59,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse; import org.apache.hadoop.yarn.api.protocolrecords.SignalContainerResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileResponse; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.CancelDelegationTokenRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.CancelDelegationTokenResponsePBImpl; @@ -120,6 +121,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.SubmitApplicationResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetAllResourceProfilesRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetAllResourceProfilesResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetAllResourceTypeInfoRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetAllResourceTypeInfoResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetResourceProfileRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetResourceProfileResponsePBImpl; import org.apache.hadoop.yarn.exceptions.YarnException; @@ -176,6 +179,8 @@ import org.apache.hadoop.yarn.proto.YarnServiceProtos.SubmitApplicationRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.SubmitApplicationResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllResourceProfilesResponseProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllResourceTypeInfoRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllResourceTypeInfoResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllResourceProfilesRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetResourceProfileRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetResourceProfileResponseProto; @@ -673,4 +678,20 @@ public GetResourceProfileResponseProto getResourceProfile( throw new ServiceException(ie); } } + + @Override + public GetAllResourceTypeInfoResponseProto getResourceTypeInfo( + RpcController controller, GetAllResourceTypeInfoRequestProto proto) + throws ServiceException { + GetAllResourceTypeInfoRequestPBImpl req = new GetAllResourceTypeInfoRequestPBImpl( + proto); + try { + GetAllResourceTypeInfoResponse resp = real.getResourceTypeInfo(req); + return ((GetAllResourceTypeInfoResponsePBImpl) resp).getProto(); + } catch (YarnException ye) { + throw new ServiceException(ye); + } catch (IOException ie) { + throw new ServiceException(ie); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllResourceTypeInfoRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllResourceTypeInfoRequestPBImpl.java new file mode 100644 index 0000000..928fc14 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllResourceTypeInfoRequestPBImpl.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoRequest; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllResourceTypeInfoRequestProto; + +/** + * Protobuf implementation class for GetAllResourceTypeInfoRequest. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +public class GetAllResourceTypeInfoRequestPBImpl + extends GetAllResourceTypeInfoRequest { + + private GetAllResourceTypeInfoRequestProto proto = + GetAllResourceTypeInfoRequestProto.getDefaultInstance(); + private GetAllResourceTypeInfoRequestProto.Builder builder = null; + + private boolean viaProto = false; + + public GetAllResourceTypeInfoRequestPBImpl() { + builder = GetAllResourceTypeInfoRequestProto.newBuilder(); + } + + public GetAllResourceTypeInfoRequestPBImpl( + GetAllResourceTypeInfoRequestProto proto) { + this.proto = proto; + viaProto = true; + } + + public GetAllResourceTypeInfoRequestProto getProto() { + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllResourceTypeInfoResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllResourceTypeInfoResponsePBImpl.java new file mode 100644 index 0000000..5bc9d06 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllResourceTypeInfoResponsePBImpl.java @@ -0,0 +1,181 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoResponse; +import org.apache.hadoop.yarn.api.records.ResourceTypeInfo; +import org.apache.hadoop.yarn.api.records.impl.pb.ResourceTypeInfoPBImpl; +import org.apache.hadoop.yarn.proto.YarnProtos.ResourceTypeInfoProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllResourceTypeInfoResponseProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllResourceTypeInfoResponseProtoOrBuilder; + +import com.google.protobuf.TextFormat; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +/** + * Protobuf implementation class for the GetAllResourceTypeInfoResponse. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +public class GetAllResourceTypeInfoResponsePBImpl + extends + GetAllResourceTypeInfoResponse { + + private GetAllResourceTypeInfoResponseProto proto = GetAllResourceTypeInfoResponseProto + .getDefaultInstance(); + private GetAllResourceTypeInfoResponseProto.Builder builder = null; + private boolean viaProto = false; + + private List resourceTypeInfo; + + public GetAllResourceTypeInfoResponsePBImpl() { + builder = GetAllResourceTypeInfoResponseProto.newBuilder(); + } + + public GetAllResourceTypeInfoResponsePBImpl( + GetAllResourceTypeInfoResponseProto proto) { + this.proto = proto; + viaProto = true; + } + + public GetAllResourceTypeInfoResponseProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public void setResourceTypeInfo(List resourceTypes) { + if (resourceTypeInfo == null) { + builder.clearResourceTypeInfo(); + } + this.resourceTypeInfo = resourceTypes; + } + + @Override + public List getResourceTypeInfo() { + initResourceTypeInfosList(); + return this.resourceTypeInfo; + } + + @Override + public boolean equals(Object other) { + if (other == null) + return false; + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } + + private void mergeLocalToBuilder() { + if (this.resourceTypeInfo != null) { + addResourceTypeInfosToProto(); + } + } + + private void mergeLocalToProto() { + if (viaProto) + maybeInitBuilder(); + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = GetAllResourceTypeInfoResponseProto.newBuilder(proto); + } + viaProto = false; + } + + // Once this is called. containerList will never be null - until a getProto + // is called. + private void initResourceTypeInfosList() { + if (this.resourceTypeInfo != null) { + return; + } + GetAllResourceTypeInfoResponseProtoOrBuilder p = viaProto ? proto : builder; + List list = p.getResourceTypeInfoList(); + resourceTypeInfo = new ArrayList(); + + for (ResourceTypeInfoProto a : list) { + resourceTypeInfo.add(convertFromProtoFormat(a)); + } + } + + private void addResourceTypeInfosToProto() { + maybeInitBuilder(); + builder.clearResourceTypeInfo(); + if (resourceTypeInfo == null) + return; + Iterable iterable = new Iterable() { + @Override + public Iterator iterator() { + return new Iterator() { + + Iterator iter = resourceTypeInfo.iterator(); + + @Override + public boolean hasNext() { + return iter.hasNext(); + } + + @Override + public ResourceTypeInfoProto next() { + return convertToProtoFormat(iter.next()); + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + + } + }; + + } + }; + builder.addAllResourceTypeInfo(iterable); + } + + private ResourceTypeInfoPBImpl convertFromProtoFormat( + ResourceTypeInfoProto p) { + return new ResourceTypeInfoPBImpl(p); + } + + private ResourceTypeInfoProto convertToProtoFormat(ResourceTypeInfo t) { + return ((ResourceTypeInfoPBImpl) t).getProto(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceTypeInfoPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceTypeInfoPBImpl.java new file mode 100644 index 0000000..75b8543 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceTypeInfoPBImpl.java @@ -0,0 +1,92 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records.impl.pb; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes; +import org.apache.hadoop.yarn.api.records.ResourceTypeInfo; +import org.apache.hadoop.yarn.proto.YarnProtos.ResourceTypeInfoProto; +import org.apache.hadoop.yarn.proto.YarnProtos.ResourceTypesProto; + +import com.google.common.base.Preconditions; + +@Private +@Unstable +public class ResourceTypeInfoPBImpl extends ResourceTypeInfo { + + ResourceTypeInfoProto proto = ResourceTypeInfoProto.getDefaultInstance(); + ResourceTypeInfoProto.Builder builder = null; + + public ResourceTypeInfoPBImpl() { + builder = ResourceTypeInfoProto.newBuilder(); + } + + public ResourceTypeInfoPBImpl(ResourceTypeInfoProto proto) { + this.proto = proto; + } + + public ResourceTypeInfoProto getProto() { + return proto; + } + + @Override + public String getName() { + Preconditions.checkNotNull(proto); + return proto.getName(); + } + + @Override + public void setName(String rName) { + Preconditions.checkNotNull(builder); + builder.setName(rName); + } + + @Override + public String getMinimumUnit() { + Preconditions.checkNotNull(proto); + return proto.getUnits(); + } + + @Override + public void setMinimumUnit(String rUnits) { + Preconditions.checkNotNull(builder); + builder.setUnits(rUnits); + } + + @Override + public ResourceTypes getResourceType() { + Preconditions.checkNotNull(proto); + return convertFromProtoFormat(proto.getType()); + } + + @Override + public void setResourceType(ResourceTypes type) { + Preconditions.checkNotNull(builder); + builder.setType(convertToProtoFormat(type)); + } + + public static ResourceTypesProto convertToProtoFormat(ResourceTypes e) { + return ResourceTypesProto.valueOf(e.name()); + } + + public static ResourceTypes convertFromProtoFormat(ResourceTypesProto e) { + return ResourceTypes.valueOf(e.name()); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/MockResourceManagerFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/MockResourceManagerFacade.java index 8cfb7f7..0f505ae 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/MockResourceManagerFacade.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/MockResourceManagerFacade.java @@ -39,6 +39,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsRequest; @@ -527,4 +529,10 @@ public GetResourceProfileResponse getResourceProfile( GetResourceProfileRequest request) throws YarnException, IOException { return null; } + + @Override + public GetAllResourceTypeInfoResponse getResourceTypeInfo( + GetAllResourceTypeInfoRequest request) throws YarnException, IOException { + return null; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index 4112624..dfdc2e8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -113,6 +113,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileResponse; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; @@ -1802,4 +1804,21 @@ public GetResourceProfileResponse getResourceProfile( } return resourceProfilesManager.getResourceProfiles(); } + + @Override + public GetAllResourceTypeInfoResponse getResourceTypeInfo( + GetAllResourceTypeInfoRequest request) throws YarnException, IOException { + boolean resourceProfilesEnabled = getConfig().getBoolean( + YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED, + YarnConfiguration.DEFAULT_RM_RESOURCE_PROFILES_ENABLED); + if (!resourceProfilesEnabled) { + throw new ResourceProfilesNotEnabledException( + "Resource profiles are not enabled"); + } + GetAllResourceTypeInfoResponse response = GetAllResourceTypeInfoResponse + .newInstance(); + response + .setResourceTypeInfo(resourceProfilesManager.getAllResourceTypeInfo()); + return response; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManager.java index af54f05..dfd5a7a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManager.java @@ -20,8 +20,10 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceTypeInfo; import java.io.IOException; +import java.util.List; import java.util.Map; /** @@ -43,4 +45,10 @@ Resource getMinimumProfile(); Resource getMaximumProfile(); + + /** + * List of ResourceTypeInfo objects which carry all resources supported by RM. + * @return list of ResourceTypeInfo objects + */ + List getAllResourceTypeInfo(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java index 7987ded..c2689a8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java @@ -24,6 +24,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceInformation; +import org.apache.hadoop.yarn.api.records.ResourceTypeInfo; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.apache.hadoop.yarn.util.resource.Resources; @@ -32,11 +33,15 @@ import java.io.File; import java.io.IOException; import java.net.URL; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.locks.ReentrantReadWriteLock; public class ResourceProfilesManagerImpl implements ResourceProfilesManager { @@ -44,6 +49,8 @@ LogFactory.getLog(ResourceProfilesManagerImpl.class); private final Map profiles = new ConcurrentHashMap<>(); + private List resourceTypeInfo = + new ArrayList(); private Configuration conf; private static final String MEMORY = ResourceInformation.MEMORY_MB.getName(); @@ -53,9 +60,18 @@ public static final String MINIMUM_PROFILE = "minimum"; public static final String MAXIMUM_PROFILE = "maximum"; + protected final ReentrantReadWriteLock.ReadLock readLock; + protected final ReentrantReadWriteLock.WriteLock writeLock; + private static final String[] MANDATORY_PROFILES = { DEFAULT_PROFILE, MINIMUM_PROFILE, MAXIMUM_PROFILE }; + public ResourceProfilesManagerImpl() { + ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); + readLock = lock.readLock(); + writeLock = lock.writeLock(); + } + public void init(Configuration config) throws IOException { conf = config; loadProfiles(); @@ -113,6 +129,20 @@ private void loadProfiles() throws IOException { + Arrays.toString(MANDATORY_PROFILES) + " must be present"); } } + // Add all resource types + try { + writeLock.lock(); + Collection resourcesInfo = ResourceUtils + .getResourceTypes().values(); + for (ResourceInformation resourceInfo : resourcesInfo) { + resourceTypeInfo + .add(ResourceTypeInfo.newInstance(resourceInfo.getName(), + resourceInfo.getUnits(), resourceInfo.getResourceType())); + } + } finally { + writeLock.unlock(); + } + LOG.info("Loaded profiles " + profiles.keySet()); } @@ -178,4 +208,13 @@ private ResourceInformation fromString(String name, String value) { Long.valueOf(value.substring(0, value.length() - units.length())); return ResourceInformation.newInstance(name, units, resourceValue); } + + public List getAllResourceTypeInfo() { + try { + readLock.lock(); + return Collections.unmodifiableList(resourceTypeInfo); + } finally { + readLock.unlock(); + } + } } -- 2.10.1 (Apple Git-78)