diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java index 803390f..8d870da 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java @@ -43,6 +43,10 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsResponse; import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteRequest; import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteResponse; import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest; @@ -430,4 +434,16 @@ public ReservationDeleteResponse deleteReservation( ReservationDeleteRequest request) throws YarnException, IOException { return client.deleteReservation(request); } + + @Override + public GetNodesToLabelsResponse getNodeToLabels() throws YarnException, + IOException { + return client.getNodeToLabels(); + } + + @Override + public GetClusterNodeLabelsResponse getClusterNodeLabels() + throws YarnException, IOException { + return client.getClusterNodeLabels(); + } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java index 5663a81..0af5a71 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java @@ -24,8 +24,6 @@ import java.net.UnknownHostException; import java.util.Iterator; -import org.junit.Assert; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -76,12 +74,14 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsResponse; -import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportRequest; @@ -92,6 +92,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoRequest; @@ -120,6 +122,7 @@ import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.ipc.YarnRPC; +import org.junit.Assert; import org.junit.Test; public class TestClientRedirect { @@ -421,6 +424,18 @@ public ReservationDeleteResponse deleteReservation( ReservationDeleteRequest request) throws YarnException, IOException { return null; } + + @Override + public GetNodesToLabelsResponse getNodeToLabels( + GetNodesToLabelsRequest request) throws YarnException, IOException { + return null; + } + + @Override + public GetClusterNodeLabelsResponse getClusterNodeLabels( + GetClusterNodeLabelsRequest request) throws YarnException, IOException { + return null; + } } class HistoryService extends AMService implements HSClientProtocol { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java index e44ddb3..add35f1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java @@ -31,12 +31,14 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsResponse; -import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportRequest; @@ -47,6 +49,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoRequest; @@ -69,6 +73,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerReport; @@ -78,9 +83,9 @@ import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.YarnClusterMetrics; -import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; -import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; +import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException; +import org.apache.hadoop.yarn.exceptions.YarnException; /** *

The protocol between clients and the ResourceManager @@ -656,4 +661,33 @@ public ReservationUpdateResponse updateReservation( public ReservationDeleteResponse deleteReservation( ReservationDeleteRequest request) throws YarnException, IOException; + /** + *

+ * The interface used by client to get node to labels mappings in existing cluster + *

+ * + * @param request + * @return node to labels mappings + * @throws YarnException + * @throws IOException + */ + @Public + @Unstable + public GetNodesToLabelsResponse getNodeToLabels( + GetNodesToLabelsRequest request) throws YarnException, IOException; + + /** + *

+ * The interface used by client to get node labels in the cluster + *

+ * + * @param request to get node labels collection of this cluster + * @return node labels collection of this cluster + * @throws YarnException + * @throws IOException + */ + @Public + @Unstable + public GetClusterNodeLabelsResponse getClusterNodeLabels( + GetClusterNodeLabelsRequest request) throws YarnException, IOException; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetClusterNodeLabelsRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetClusterNodeLabelsRequest.java new file mode 100644 index 0000000..c1736aa --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetClusterNodeLabelsRequest.java @@ -0,0 +1,31 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Evolving; +import org.apache.hadoop.yarn.util.Records; + +@Public +@Evolving +public abstract class GetClusterNodeLabelsRequest { + public static GetClusterNodeLabelsRequest newInstance() { + return Records.newRecord(GetClusterNodeLabelsRequest.class); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetClusterNodeLabelsResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetClusterNodeLabelsResponse.java new file mode 100644 index 0000000..7bd8223 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetClusterNodeLabelsResponse.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords; + +import java.util.Set; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Evolving; +import org.apache.hadoop.yarn.util.Records; + +@Public +@Evolving +public abstract class GetClusterNodeLabelsResponse { + public static GetClusterNodeLabelsResponse newInstance(Set labels) { + GetClusterNodeLabelsResponse request = + Records.newRecord(GetClusterNodeLabelsResponse.class); + request.setNodeLabels(labels); + return request; + } + + @Public + @Evolving + public abstract void setNodeLabels(Set labels); + + @Public + @Evolving + public abstract Set getNodeLabels(); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetNodesToLabelsRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetNodesToLabelsRequest.java new file mode 100644 index 0000000..9ca4c91 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetNodesToLabelsRequest.java @@ -0,0 +1,27 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.api.protocolrecords; + +import org.apache.hadoop.yarn.util.Records; + +public abstract class GetNodesToLabelsRequest { + public static GetNodesToLabelsRequest newInstance() { + return Records.newRecord(GetNodesToLabelsRequest.class); + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetNodesToLabelsResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetNodesToLabelsResponse.java new file mode 100644 index 0000000..bcd5421 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetNodesToLabelsResponse.java @@ -0,0 +1,45 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.api.protocolrecords; + +import java.util.Map; +import java.util.Set; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Evolving; +import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.util.Records; + +public abstract class GetNodesToLabelsResponse { + public static GetNodesToLabelsResponse newInstance( + Map> map) { + GetNodesToLabelsResponse response = + Records.newRecord(GetNodesToLabelsResponse.class); + response.setNodeToLabels(map); + return response; + } + + @Public + @Evolving + public abstract void setNodeToLabels(Map> map); + + @Public + @Evolving + public abstract Map> getNodeToLabels(); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java index 36d851c..2061aef 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java @@ -32,10 +32,6 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsResponse; -import org.apache.hadoop.yarn.server.api.protocolrecords.GetClusterNodeLabelsRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.GetClusterNodeLabelsResponse; -import org.apache.hadoop.yarn.server.api.protocolrecords.GetNodesToLabelsRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.GetNodesToLabelsResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesRequest; @@ -138,16 +134,4 @@ public RemoveFromClusterNodeLabelsResponse removeFromClusterNodeLabels( @Idempotent public ReplaceLabelsOnNodeResponse replaceLabelsOnNode( ReplaceLabelsOnNodeRequest request) throws YarnException, IOException; - - @Public - @Evolving - @Idempotent - public GetNodesToLabelsResponse getNodeToLabels( - GetNodesToLabelsRequest request) throws YarnException, IOException; - - @Public - @Evolving - @Idempotent - public GetClusterNodeLabelsResponse getClusterNodeLabels( - GetClusterNodeLabelsRequest request) throws YarnException, IOException; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/GetClusterNodeLabelsRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/GetClusterNodeLabelsRequest.java deleted file mode 100644 index 208fe77..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/GetClusterNodeLabelsRequest.java +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.api.protocolrecords; - -import org.apache.hadoop.classification.InterfaceAudience.Public; -import org.apache.hadoop.classification.InterfaceStability.Evolving; -import org.apache.hadoop.yarn.util.Records; - -@Public -@Evolving -public abstract class GetClusterNodeLabelsRequest { - public static GetClusterNodeLabelsRequest newInstance() { - return Records.newRecord(GetClusterNodeLabelsRequest.class); - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/GetClusterNodeLabelsResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/GetClusterNodeLabelsResponse.java deleted file mode 100644 index c585d53..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/GetClusterNodeLabelsResponse.java +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.api.protocolrecords; - -import java.util.Set; - -import org.apache.hadoop.classification.InterfaceAudience.Public; -import org.apache.hadoop.classification.InterfaceStability.Evolving; -import org.apache.hadoop.yarn.util.Records; - -@Public -@Evolving -public abstract class GetClusterNodeLabelsResponse { - public static GetClusterNodeLabelsResponse newInstance(Set labels) { - GetClusterNodeLabelsResponse request = - Records.newRecord(GetClusterNodeLabelsResponse.class); - request.setNodeLabels(labels); - return request; - } - - @Public - @Evolving - public abstract void setNodeLabels(Set labels); - - @Public - @Evolving - public abstract Set getNodeLabels(); -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/GetNodesToLabelsRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/GetNodesToLabelsRequest.java deleted file mode 100644 index b2efee2..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/GetNodesToLabelsRequest.java +++ /dev/null @@ -1,27 +0,0 @@ -/** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -package org.apache.hadoop.yarn.server.api.protocolrecords; - -import org.apache.hadoop.yarn.util.Records; - -public abstract class GetNodesToLabelsRequest { - public static GetNodesToLabelsRequest newInstance() { - return Records.newRecord(GetNodesToLabelsRequest.class); - } -} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/GetNodesToLabelsResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/GetNodesToLabelsResponse.java deleted file mode 100644 index 9c4aeb2..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/GetNodesToLabelsResponse.java +++ /dev/null @@ -1,45 +0,0 @@ -/** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -package org.apache.hadoop.yarn.server.api.protocolrecords; - -import java.util.Map; -import java.util.Set; - -import org.apache.hadoop.classification.InterfaceAudience.Public; -import org.apache.hadoop.classification.InterfaceStability.Evolving; -import org.apache.hadoop.yarn.api.records.NodeId; -import org.apache.hadoop.yarn.util.Records; - -public abstract class GetNodesToLabelsResponse { - public static GetNodesToLabelsResponse newInstance( - Map> map) { - GetNodesToLabelsResponse response = - Records.newRecord(GetNodesToLabelsResponse.class); - response.setNodeToLabels(map); - return response; - } - - @Public - @Evolving - public abstract void setNodeToLabels(Map> map); - - @Public - @Evolving - public abstract Map> getNodeToLabels(); -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/applicationclient_protocol.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/applicationclient_protocol.proto index 3346aef..8e788cd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/applicationclient_protocol.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/applicationclient_protocol.proto @@ -52,5 +52,6 @@ service ApplicationClientProtocolService { rpc submitReservation (ReservationSubmissionRequestProto) returns (ReservationSubmissionResponseProto); rpc updateReservation (ReservationUpdateRequestProto) returns (ReservationUpdateResponseProto); rpc deleteReservation (ReservationDeleteRequestProto) returns (ReservationDeleteResponseProto); + rpc getNodeToLabels (GetNodesToLabelsRequestProto) returns (GetNodesToLabelsResponseProto); + rpc getClusterNodeLabels (GetClusterNodeLabelsRequestProto) returns (GetClusterNodeLabelsResponseProto); } - diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/resourcemanager_administration_protocol.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/resourcemanager_administration_protocol.proto index 99dd4fd..7f54b8e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/resourcemanager_administration_protocol.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/resourcemanager_administration_protocol.proto @@ -42,6 +42,4 @@ service ResourceManagerAdministrationProtocolService { rpc addToClusterNodeLabels(AddToClusterNodeLabelsRequestProto) returns (AddToClusterNodeLabelsResponseProto); rpc removeFromClusterNodeLabels(RemoveFromClusterNodeLabelsRequestProto) returns (RemoveFromClusterNodeLabelsResponseProto); rpc replaceLabelsOnNodes(ReplaceLabelsOnNodeRequestProto) returns (ReplaceLabelsOnNodeResponseProto); - rpc getNodeToLabels(GetNodesToLabelsRequestProto) returns (GetNodesToLabelsResponseProto); - rpc getClusterNodeLabels(GetClusterNodeLabelsRequestProto) returns (GetClusterNodeLabelsResponseProto); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto index 5c63cfe..900e349 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto @@ -89,11 +89,6 @@ message RemoveFromClusterNodeLabelsRequestProto { message RemoveFromClusterNodeLabelsResponseProto { } -message NodeIdToLabelsProto { - optional NodeIdProto nodeId = 1; - repeated string nodeLabels = 2; -} - message ReplaceLabelsOnNodeRequestProto { repeated NodeIdToLabelsProto nodeToLabels = 1; } @@ -102,20 +97,6 @@ message ReplaceLabelsOnNodeResponseProto { } -message GetNodesToLabelsRequestProto { -} - -message GetNodesToLabelsResponseProto { - repeated NodeIdToLabelsProto nodeToLabels = 1; -} - -message GetClusterNodeLabelsRequestProto { -} - -message GetClusterNodeLabelsResponseProto { - repeated string nodeLabels = 1; -} - ////////////////////////////////////////////////////////////////// ///////////// RM Failover related records //////////////////////// ////////////////////////////////////////////////////////////////// diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 136192d..71a915d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -231,6 +231,10 @@ message NodeReportProto { optional int64 last_health_report_time = 9; } +message NodeIdToLabelsProto { + optional NodeIdProto nodeId = 1; + repeated string nodeLabels = 2; +} //////////////////////////////////////////////////////////////////////// ////// From AM_RM_Protocol ///////////////////////////////////////////// diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto index 2585f78..1bde69a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto @@ -194,6 +194,19 @@ message GetQueueUserAclsInfoResponseProto { repeated QueueUserACLInfoProto queueUserAcls = 1; } +message GetNodesToLabelsRequestProto { +} + +message GetNodesToLabelsResponseProto { + repeated NodeIdToLabelsProto nodeToLabels = 1; +} + +message GetClusterNodeLabelsRequestProto { +} + +message GetClusterNodeLabelsResponseProto { + repeated string nodeLabels = 1; +} ////////////////////////////////////////////////////// /////// client_NM_Protocol /////////////////////////// diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java index 7cc6dde..6b96ea1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java @@ -32,6 +32,10 @@ import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsResponse; import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteRequest; import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteResponse; import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest; @@ -579,4 +583,32 @@ public abstract ReservationUpdateResponse updateReservation( @Unstable public abstract ReservationDeleteResponse deleteReservation( ReservationDeleteRequest request) throws YarnException, IOException; + + /** + *

+ * The interface used by client to get node to labels mappings in existing cluster + *

+ * + * @return node to labels mappings + * @throws YarnException + * @throws IOException + */ + @Public + @Unstable + public abstract GetNodesToLabelsResponse getNodeToLabels() + throws YarnException, IOException; + + /** + *

+ * The interface used by client to get node labels in the cluster + *

+ * + * @return cluster node labels collection + * @throws YarnException + * @throws IOException + */ + @Public + @Unstable + public abstract GetClusterNodeLabelsResponse getClusterNodeLabels() + throws YarnException, IOException; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java index 02c5a74..1e80049 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java @@ -28,6 +28,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.DataInputByteBuffer; @@ -48,6 +49,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportRequest; @@ -58,6 +61,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoRequest; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest; @@ -670,5 +675,17 @@ public ReservationDeleteResponse deleteReservation( ReservationDeleteRequest request) throws YarnException, IOException { return rmClient.deleteReservation(request); } + + @Override + public GetNodesToLabelsResponse getNodeToLabels() throws YarnException, + IOException { + return rmClient.getNodeToLabels(GetNodesToLabelsRequest.newInstance()); + } + @Override + public GetClusterNodeLabelsResponse getClusterNodeLabels() + throws YarnException, IOException { + return rmClient.getClusterNodeLabels(GetClusterNodeLabelsRequest + .newInstance()); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeLabelsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeLabelsCLI.java new file mode 100644 index 0000000..7badf4e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeLabelsCLI.java @@ -0,0 +1,212 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.client.cli; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.GnuParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.MissingArgumentException; +import org.apache.commons.cli.Options; +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.util.ToolRunner; +import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; + +import com.google.common.annotations.VisibleForTesting; + +/** + * YARN Client used to get labels and node-to-label mappings of the cluster + */ +public class NodeLabelsCLI extends YarnCLI { + private static final String NODE_ID_CMD = "nodeId"; + private static final String TITLE = "node-labels"; + private static final String ALL_NODE_ID = "-all"; + private static final String DIRECTLY_ACCESS_NODE_LABEL_STORE = + "directlyAccessNodeLabelStore"; + private boolean accessLocal = false; + static CommonNodeLabelsManager localNodeLabelsManager = null; + + public static void main(String[] args) throws Exception { + NodeLabelsCLI cli = new NodeLabelsCLI(); + cli.setSysOutPrintStream(System.out); + cli.setSysErrPrintStream(System.err); + int res = ToolRunner.run(cli, args); + cli.stop(); + System.exit(res); + } + + @Override + public int run(String[] args) throws Exception { + Options opts = new Options(); + + if (args.length == 0 || !args[0].equalsIgnoreCase(TITLE)) { + throw new IOException("Command should start with: " + TITLE); + } + + opts.addOption(LIST_CMD, false, "List node labels " + "If -" + NODE_ID_CMD + + " not specifed, will return all labels in the cluster. If -" + + NODE_ID_CMD + " specified, will return node to label mappings"); + opts.addOption(NODE_ID_CMD, true, "Works with -" + LIST_CMD + + ", it will return node to label mappings when this specified"); + opts.addOption(HELP_CMD, false, "Displays help for all commands."); + opts.addOption(DIRECTLY_ACCESS_NODE_LABEL_STORE, false, + "Directly access node label store, " + + "with this option, all node label related operations" + + " will NOT connect RM. Instead, they will" + + " access/modify stored node labels directly." + + " By default, it is false (access via RM)." + + " AND PLEASE NOTE: if you configured" + + " yarn.node-labels.fs-store.uri to a local directory" + + " (instead of NFS or HDFS), this option will only work" + + " when the command run on the machine where RM is running."); + + int exitCode = -1; + CommandLine cliParser = null; + try { + cliParser = new GnuParser().parse(opts, args); + } catch (MissingArgumentException ex) { + sysout.println("Missing argument for options"); + printUsage(opts); + return exitCode; + } + + if (cliParser.hasOption(DIRECTLY_ACCESS_NODE_LABEL_STORE)) { + accessLocal = true; + } + + if (cliParser.hasOption(LIST_CMD)) { + if (cliParser.hasOption(NODE_ID_CMD)) { + // print node to labels mapping + String nodeId = cliParser.getOptionValue(NODE_ID_CMD); + printNodeToLabelsMapping(nodeId); + } else { + printClusterNodeLabels(); + } + return 0; + } else if (cliParser.hasOption(HELP_CMD)) { + printUsage(opts); + return 0; + } else { + syserr.println("Invalid Command Usage : "); + printUsage(opts); + } + return 0; + } + + private List sortNodeIdSet(Set nodes) { + List list = new ArrayList(); + list.addAll(nodes); + Collections.sort(list); + return list; + } + + private List sortStrSet(Set labels) { + List list = new ArrayList(); + list.addAll(labels); + Collections.sort(list); + return list; + } + + private NodeId buildNodeIdFromStr(String str) { + String host = str; + int port = 0; + if (str.contains(":")) { + port = Integer.valueOf(StringUtils.substring(str, str.indexOf(':') + 1)); + host = StringUtils.substring(str, 0, str.indexOf(':')); + } + return NodeId.newInstance(host, port); + } + + private void printSingleNodeToLabelsMapping(PrintWriter pw, NodeId nodeId, + Map> nodeIdToLabels) { + StringBuilder sb = new StringBuilder(); + sb.append(String.format("Host=%s, Node-labels=", + (nodeId.getPort() == 0 ? nodeId.getHost() : nodeId.toString()))); + + if (nodeIdToLabels.containsKey(nodeId)) { + sb.append(StringUtils.join(sortStrSet(nodeIdToLabels.get(nodeId)), ",")); + } + pw.println(sb.toString()); + } + + void printNodeToLabelsMapping(String nodeIdStr) throws YarnException, + IOException { + Map> nodeIdToLabels = null; + + if (accessLocal) { + nodeIdToLabels = getNodeLabelManagerInstance(getConf()).getNodeLabels(); + } else { + nodeIdToLabels = client.getNodeToLabels().getNodeToLabels(); + } + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintWriter pw = new PrintWriter(baos); + + if (ALL_NODE_ID.equalsIgnoreCase(nodeIdStr)) { + for (NodeId host : sortNodeIdSet(nodeIdToLabels.keySet())) { + printSingleNodeToLabelsMapping(pw, host, nodeIdToLabels); + } + } else { + NodeId host = buildNodeIdFromStr(nodeIdStr); + printSingleNodeToLabelsMapping(pw, host, nodeIdToLabels); + } + + pw.close(); + sysout.println(baos.toString("UTF-8")); + } + + void printClusterNodeLabels() throws YarnException, IOException { + Set nodeLabels = null; + if (accessLocal) { + nodeLabels = + getNodeLabelManagerInstance(getConf()).getClusterNodeLabels(); + } else { + nodeLabels = client.getClusterNodeLabels().getNodeLabels(); + } + sysout.println(String.format("Node-labels=%s", + StringUtils.join(sortStrSet(nodeLabels).iterator(), ","))); + } + + // Make it protected to make unit test can change it. + protected static synchronized CommonNodeLabelsManager + getNodeLabelManagerInstance(Configuration conf) { + if (localNodeLabelsManager == null) { + localNodeLabelsManager = new CommonNodeLabelsManager(); + localNodeLabelsManager.init(conf); + localNodeLabelsManager.start(); + } + return localNodeLabelsManager; + } + + @VisibleForTesting + void printUsage(Options opts) { + new HelpFormatter().printHelp(TITLE, opts); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java index cd884f2..4a413f9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java @@ -22,14 +22,12 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; -import org.apache.commons.lang.StringUtils; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; @@ -51,8 +49,6 @@ import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocol; import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.GetClusterNodeLabelsRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.GetNodesToLabelsRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshQueuesRequest; @@ -104,10 +100,6 @@ .put("-replaceLabelsOnNode", new UsageInfo("[node1:port,label1,label2 node2:port,label1,label2]", "replace labels on nodes")) - .put("-getNodeToLabels", new UsageInfo("", - "Get node to label mappings")) - .put("-getClusterNodeLabels", - new UsageInfo("", "Get node labels in the cluster")) .put("-directlyAccessNodeLabelStore", new UsageInfo("", "Directly access node label store, " + "with this option, all node label related operations" @@ -117,7 +109,8 @@ + " AND PLEASE NOTE: if you configured" + " yarn.node-labels.fs-store.uri to a local directory" + " (instead of NFS or HDFS), this option will only work" - + " when the command run on the machine where RM is running.")) + + + " when the command run on the machine where RM is running.")) .build(); public RMAdminCLI() { @@ -368,9 +361,10 @@ private int removeFromClusterNodeLabels(String args) throws IOException, for (String p : args.split(",")) { labels.add(p); } - + if (directlyAccessNodeLabelStore) { - getNodeLabelManagerInstance(getConf()).removeFromClusterNodeLabels(labels); + getNodeLabelManagerInstance(getConf()).removeFromClusterNodeLabels( + labels); } else { ResourceManagerAdministrationProtocol adminProtocol = createAdminProtocol(); @@ -381,57 +375,6 @@ private int removeFromClusterNodeLabels(String args) throws IOException, return 0; } - - private int getNodeToLabels() throws IOException, YarnException { - Map> nodeToLabels = null; - - if (directlyAccessNodeLabelStore) { - nodeToLabels = getNodeLabelManagerInstance(getConf()).getNodeLabels(); - } else { - ResourceManagerAdministrationProtocol adminProtocol = - createAdminProtocol(); - - nodeToLabels = - adminProtocol.getNodeToLabels(GetNodesToLabelsRequest.newInstance()) - .getNodeToLabels(); - } - for (NodeId host : sortNodeIdSet(nodeToLabels.keySet())) { - System.out.println(String.format("Host=%s, Node-labels=[%s]", - (host.getPort() == 0 ? host.getHost() : host.toString()), - StringUtils.join(sortStrSet(nodeToLabels.get(host)), ","))); - } - return 0; - } - - private int getClusterNodeLabels() throws IOException, YarnException { - Set labels = null; - if (directlyAccessNodeLabelStore) { - labels = getNodeLabelManagerInstance(getConf()).getClusterNodeLabels(); - } else { - ResourceManagerAdministrationProtocol adminProto = createAdminProtocol(); - labels = - adminProto.getClusterNodeLabels( - GetClusterNodeLabelsRequest.newInstance()).getNodeLabels(); - } - - System.out.println(String.format("Node-labels=%s", - StringUtils.join(sortStrSet(labels).iterator(), ","))); - return 0; - } - - private List sortNodeIdSet(Set nodes) { - List list = new ArrayList(); - list.addAll(nodes); - Collections.sort(list); - return list; - } - - private List sortStrSet(Set labels) { - List list = new ArrayList(); - list.addAll(labels); - Collections.sort(list); - return list; - } private Map> buildNodeLabelsFromStr(String args) throws IOException { @@ -593,10 +536,6 @@ public int run(String[] args) throws Exception { } else { exitCode = replaceLabelsOnNodes(args[i]); } - } else if ("-getNodeToLabels".equals(cmd)) { - exitCode = getNodeToLabels(); - } else if ("-getClusterNodeLabels".equals(cmd)) { - exitCode = getClusterNodeLabels(); } else { exitCode = -1; System.err.println(cmd.substring(1) + ": Unknown command"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java index 4cabeb7..6176a3e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java @@ -502,24 +502,6 @@ public void testReplaceLabelsOnNodeWithPort() throws Exception { assertTrue(dummyNodeLabelsManager.getNodeLabels().containsKey( NodeId.newInstance("node2", 8000))); } - - @Test - public void testGetClusterNodeLabels() throws Exception { - // Successfully get labels - String[] args = - { "-getClusterNodeLabels", - "-directlyAccessNodeLabelStore" }; - assertEquals(0, rmAdminCLI.run(args)); - } - - @Test - public void testGetNodeToLabels() throws Exception { - // Successfully get node-to-labels - String[] args = - { "-getNodeToLabels", - "-directlyAccessNodeLabelStore" }; - assertEquals(0, rmAdminCLI.run(args)); - } private void testError(String[] args, String template, ByteArrayOutputStream data, int resultCode) throws Exception { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java index 980517f..0c76bda 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java @@ -21,12 +21,12 @@ import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyInt; import static org.mockito.Matchers.isA; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import static org.mockito.Mockito.doThrow; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -35,13 +35,16 @@ import java.util.ArrayList; import java.util.Date; import java.util.EnumSet; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; -import org.junit.Assert; - +import org.apache.commons.cli.Options; import org.apache.commons.lang.time.DateFormatUtils; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsResponse; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -60,12 +63,15 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; +import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; import org.apache.hadoop.yarn.util.Records; +import org.jboss.netty.logging.CommonsLoggerFactory; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.mortbay.log.Log; -import org.apache.commons.cli.Options; +import com.google.common.collect.ImmutableSet; public class TestYarnCLI { @@ -1194,6 +1200,215 @@ public void testMissingArguments() throws Exception { Assert.assertEquals(String.format("Missing argument for options%n%1s", createNodeCLIHelpMessage()), sysOutStream.toString()); } + + @Test + public void testGetClusterNodeLabels() throws Exception { + YarnClient client = mock(YarnClient.class); + when(client.getClusterNodeLabels()).thenReturn( + GetClusterNodeLabelsResponse.newInstance(ImmutableSet.of("label1", + "label2"))); + NodeLabelsCLI cli = new NodeLabelsCLI(); + cli.setClient(client); + cli.setSysOutPrintStream(sysOut); + cli.setSysErrPrintStream(sysErr); + + int rc = cli.run(new String[] { "node-labels", "-list" } ); + assertEquals(0, rc); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintWriter pw = new PrintWriter(baos); + pw.print("Node-labels=label1,label2"); + pw.close(); + verify(sysOut).println(baos.toString("UTF-8")); + } + + @Test + public void testGetClusterNodeLabelsWithLocalAccess() throws Exception { + YarnClient client = mock(YarnClient.class); + when(client.getClusterNodeLabels()).thenReturn( + GetClusterNodeLabelsResponse.newInstance(ImmutableSet.of("remote1", + "remote2"))); + NodeLabelsCLI cli = new NodeLabelsCLI(); + cli.setClient(client); + cli.setSysOutPrintStream(sysOut); + cli.setSysErrPrintStream(sysErr); + NodeLabelsCLI.localNodeLabelsManager = mock(CommonNodeLabelsManager.class); + when(NodeLabelsCLI.localNodeLabelsManager.getClusterNodeLabels()) + .thenReturn(ImmutableSet.of("local1", "local2")); + + int rc = + cli.run(new String[] { "node-labels", "-list", + "-directlyAccessNodeLabelStore" }); + assertEquals(0, rc); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintWriter pw = new PrintWriter(baos); + // it should return local* instead of remote* + pw.print("Node-labels=local1,local2"); + pw.close(); + verify(sysOut).println(baos.toString("UTF-8")); + } + + @Test + public void testGetEmptyClusterNodeLabels() throws Exception { + YarnClient client = mock(YarnClient.class); + when(client.getClusterNodeLabels()).thenReturn( + GetClusterNodeLabelsResponse.newInstance(new HashSet())); + NodeLabelsCLI cli = new NodeLabelsCLI(); + cli.setClient(client); + cli.setSysOutPrintStream(sysOut); + cli.setSysErrPrintStream(sysErr); + + int rc = cli.run(new String[] { "node-labels", "-list" } ); + assertEquals(0, rc); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintWriter pw = new PrintWriter(baos); + pw.print("Node-labels="); + pw.close(); + verify(sysOut).println(baos.toString("UTF-8")); + } + + @Test + public void testGetAllNodeToLabels() throws Exception { + YarnClient client = mock(YarnClient.class); + Map> map = new HashMap>(); + map.put(NodeId.newInstance("node1", 0), ImmutableSet.of("label1", "label2")); + map.put(NodeId.newInstance("node2", 0), ImmutableSet.of("label3", "label4")); + map.put(NodeId.newInstance("node1", 123), + ImmutableSet.of("label5", "label6")); + + when(client.getNodeToLabels()).thenReturn( + GetNodesToLabelsResponse.newInstance(map)); + NodeLabelsCLI cli = new NodeLabelsCLI(); + cli.setClient(client); + cli.setSysOutPrintStream(sysOut); + cli.setSysErrPrintStream(sysErr); + + int rc = cli.run(new String[] { "node-labels", "-list", "-nodeId", "-all" } ); + assertEquals(0, rc); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintWriter pw = new PrintWriter(baos); + pw.println("Host=node1, Node-labels=label1,label2"); + pw.println("Host=node1:123, Node-labels=label5,label6"); + pw.println("Host=node2, Node-labels=label3,label4"); + pw.close(); + verify(sysOut).println(baos.toString("UTF-8")); + } + + @Test + public void testGetNodeToLabelsWithSpecificHost() throws Exception { + YarnClient client = mock(YarnClient.class); + Map> map = new HashMap>(); + map.put(NodeId.newInstance("node1", 0), ImmutableSet.of("label1", "label2")); + map.put(NodeId.newInstance("node2", 0), ImmutableSet.of("label3", "label4")); + map.put(NodeId.newInstance("node1", 123), + ImmutableSet.of("label5", "label6")); + + when(client.getNodeToLabels()).thenReturn( + GetNodesToLabelsResponse.newInstance(map)); + NodeLabelsCLI cli = new NodeLabelsCLI(); + cli.setClient(client); + cli.setSysOutPrintStream(sysOut); + cli.setSysErrPrintStream(sysErr); + + int rc = cli.run(new String[] { "node-labels", "-list", "-nodeId", "node1" } ); + assertEquals(0, rc); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintWriter pw = new PrintWriter(baos); + pw.println("Host=node1, Node-labels=label1,label2"); + pw.close(); + verify(sysOut).println(baos.toString("UTF-8")); + } + + @Test + public void testGetNodeToLabelsWithSpecificNode() throws Exception { + YarnClient client = mock(YarnClient.class); + Map> map = new HashMap>(); + map.put(NodeId.newInstance("node1", 0), ImmutableSet.of("label1", "label2")); + map.put(NodeId.newInstance("node2", 0), ImmutableSet.of("label3", "label4")); + map.put(NodeId.newInstance("node1", 123), + ImmutableSet.of("label5", "label6")); + + when(client.getNodeToLabels()).thenReturn( + GetNodesToLabelsResponse.newInstance(map)); + NodeLabelsCLI cli = new NodeLabelsCLI(); + cli.setClient(client); + cli.setSysOutPrintStream(sysOut); + cli.setSysErrPrintStream(sysErr); + + int rc = + cli.run(new String[] { "node-labels", "-list", "-nodeId", "node1:123" }); + assertEquals(0, rc); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintWriter pw = new PrintWriter(baos); + pw.println("Host=node1:123, Node-labels=label5,label6"); + pw.close(); + verify(sysOut).println(baos.toString("UTF-8")); + } + + @Test + public void testGetNodeToLabelsWithSpecificNotExistedNode() throws Exception { + YarnClient client = mock(YarnClient.class); + Map> map = new HashMap>(); + map.put(NodeId.newInstance("node1", 0), ImmutableSet.of("label1", "label2")); + map.put(NodeId.newInstance("node2", 0), ImmutableSet.of("label3", "label4")); + map.put(NodeId.newInstance("node1", 123), + ImmutableSet.of("label5", "label6")); + + when(client.getNodeToLabels()).thenReturn( + GetNodesToLabelsResponse.newInstance(map)); + NodeLabelsCLI cli = new NodeLabelsCLI(); + cli.setClient(client); + cli.setSysOutPrintStream(sysOut); + cli.setSysErrPrintStream(sysErr); + + // node 3 isn't existed .. + int rc = + cli.run(new String[] { "node-labels", "-list", "-nodeId", "node3" }); + assertEquals(0, rc); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintWriter pw = new PrintWriter(baos); + pw.println("Host=node3, Node-labels="); + pw.close(); + verify(sysOut).println(baos.toString("UTF-8")); + } + + @Test + public void testGetNodeToLabelsWithLocalAccess() throws Exception { + YarnClient client = mock(YarnClient.class); + Map> map = new HashMap>(); + map.put(NodeId.newInstance("remote1", 0), + ImmutableSet.of("label1", "label2")); + when(client.getNodeToLabels()).thenReturn( + GetNodesToLabelsResponse.newInstance(map)); + map = new HashMap>(); + map.put(NodeId.newInstance("local1", 0), + ImmutableSet.of("label1", "label2")); + + NodeLabelsCLI cli = new NodeLabelsCLI(); + cli.setClient(client); + cli.setSysOutPrintStream(sysOut); + cli.setSysErrPrintStream(sysErr); + NodeLabelsCLI.localNodeLabelsManager = mock(CommonNodeLabelsManager.class); + when(NodeLabelsCLI.localNodeLabelsManager.getNodeLabels()).thenReturn(map); + + // node 3 isn't existed .. + int rc = + cli.run(new String[] { "node-labels", "-list", "-nodeId", "-all", + "-directlyAccessNodeLabelStore" }); + assertEquals(0, rc); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintWriter pw = new PrintWriter(baos); + pw.println("Host=local1, Node-labels=label1,label2"); + pw.close(); + verify(sysOut).println(baos.toString("UTF-8")); + } private void verifyUsageInfo(YarnCLI cli) throws Exception { cli.setSysErrPrintStream(sysErr); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ApplicationClientProtocolPBClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ApplicationClientProtocolPBClientImpl.java index b4b9f43..0fe5423 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ApplicationClientProtocolPBClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ApplicationClientProtocolPBClientImpl.java @@ -37,12 +37,14 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsResponse; -import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportRequest; @@ -53,6 +55,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoRequest; @@ -77,12 +81,14 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationAttemptReportResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationAttemptsRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationAttemptsResponsePBImpl; -import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationsRequestPBImpl; -import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationsResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationReportRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationReportResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationsRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationsResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterMetricsRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterMetricsResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodeLabelsRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodeLabelsResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodesRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodesResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetContainerReportRequestPBImpl; @@ -93,6 +99,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetDelegationTokenResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetNewApplicationRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetNewApplicationResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetNodesToLabelsRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetNodesToLabelsResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetQueueInfoRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetQueueInfoResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetQueueUserAclsInfoRequestPBImpl; @@ -113,23 +121,25 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.SubmitApplicationResponsePBImpl; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.ipc.RPCUtil; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationsRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationAttemptReportRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationAttemptsRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationReportRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationsRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetClusterMetricsRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetClusterNodeLabelsRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetClusterNodesRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetContainerReportRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetContainersRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetNewApplicationRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetQueueInfoRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetQueueUserAclsInfoRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.KillApplicationRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.MoveApplicationAcrossQueuesRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.ReservationDeleteRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.ReservationSubmissionRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.ReservationUpdateRequestProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.ReservationDeleteRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.SubmitApplicationRequestProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationAttemptsRequestProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationAttemptReportRequestProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetContainerReportRequestProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetContainersRequestProto; import com.google.protobuf.ServiceException; @@ -447,4 +457,35 @@ public ReservationDeleteResponse deleteReservation(ReservationDeleteRequest requ } } + + @Override + public GetNodesToLabelsResponse getNodeToLabels( + GetNodesToLabelsRequest request) + throws YarnException, IOException { + YarnServiceProtos.GetNodesToLabelsRequestProto + requestProto = + ((GetNodesToLabelsRequestPBImpl) request).getProto(); + try { + return new GetNodesToLabelsResponsePBImpl(proxy.getNodeToLabels( + null, requestProto)); + } catch (ServiceException e) { + RPCUtil.unwrapAndThrowException(e); + return null; + } + } + + @Override + public GetClusterNodeLabelsResponse getClusterNodeLabels( + GetClusterNodeLabelsRequest request) throws YarnException, IOException { + GetClusterNodeLabelsRequestProto + requestProto = + ((GetClusterNodeLabelsRequestPBImpl) request).getProto(); + try { + return new GetClusterNodeLabelsResponsePBImpl(proxy.getClusterNodeLabels( + null, requestProto)); + } catch (ServiceException e) { + RPCUtil.unwrapAndThrowException(e); + return null; + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ApplicationClientProtocolPBServiceImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ApplicationClientProtocolPBServiceImpl.java index f73202d..385fb78 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ApplicationClientProtocolPBServiceImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ApplicationClientProtocolPBServiceImpl.java @@ -32,14 +32,16 @@ import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsResponse; -import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetContainersResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoResponse; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse; @@ -55,12 +57,14 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationAttemptReportResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationAttemptsRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationAttemptsResponsePBImpl; -import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationsRequestPBImpl; -import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationsResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationReportRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationReportResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationsRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationsResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterMetricsRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterMetricsResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodeLabelsRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodeLabelsResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodesRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodesResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetContainerReportRequestPBImpl; @@ -71,6 +75,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetDelegationTokenResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetNewApplicationRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetNewApplicationResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetNodesToLabelsRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetNodesToLabelsResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetQueueInfoRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetQueueInfoResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetQueueUserAclsInfoRequestPBImpl; @@ -90,16 +96,28 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.SubmitApplicationRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.SubmitApplicationResponsePBImpl; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationsRequestProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationsResponseProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationAttemptReportRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationAttemptReportResponseProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationAttemptsRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationAttemptsResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationReportRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationReportResponseProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationsRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationsResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetClusterMetricsRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetClusterMetricsResponseProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetClusterNodeLabelsRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetClusterNodeLabelsResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetClusterNodesRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetClusterNodesResponseProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetContainerReportRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetContainerReportResponseProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetContainersRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetContainersResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetNewApplicationRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetNewApplicationResponseProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetNodesToLabelsRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetNodesToLabelsResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetQueueInfoRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetQueueInfoResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetQueueUserAclsInfoRequestProto; @@ -108,22 +126,14 @@ import org.apache.hadoop.yarn.proto.YarnServiceProtos.KillApplicationResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.MoveApplicationAcrossQueuesRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.MoveApplicationAcrossQueuesResponseProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.ReservationDeleteRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.ReservationDeleteResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.ReservationSubmissionRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.ReservationSubmissionResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.ReservationUpdateRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.ReservationUpdateResponseProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.ReservationDeleteRequestProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.ReservationDeleteResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.SubmitApplicationRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.SubmitApplicationResponseProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationAttemptReportRequestProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationAttemptReportResponseProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationAttemptsRequestProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationAttemptsResponseProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetContainerReportRequestProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetContainerReportResponseProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetContainersRequestProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetContainersResponseProto; import com.google.protobuf.RpcController; import com.google.protobuf.ServiceException; @@ -444,4 +454,36 @@ public ReservationDeleteResponseProto deleteReservation(RpcController controller } } + @Override + public GetNodesToLabelsResponseProto getNodeToLabels( + RpcController controller, GetNodesToLabelsRequestProto proto) + throws ServiceException { + GetNodesToLabelsRequestPBImpl request = + new GetNodesToLabelsRequestPBImpl(proto); + try { + GetNodesToLabelsResponse response = real.getNodeToLabels(request); + return ((GetNodesToLabelsResponsePBImpl) response).getProto(); + } catch (YarnException e) { + throw new ServiceException(e); + } catch (IOException e) { + throw new ServiceException(e); + } + } + + @Override + public GetClusterNodeLabelsResponseProto getClusterNodeLabels( + RpcController controller, GetClusterNodeLabelsRequestProto proto) + throws ServiceException { + GetClusterNodeLabelsRequestPBImpl request = + new GetClusterNodeLabelsRequestPBImpl(proto); + try { + GetClusterNodeLabelsResponse response = + real.getClusterNodeLabels(request); + return ((GetClusterNodeLabelsResponsePBImpl) response).getProto(); + } catch (YarnException e) { + throw new ServiceException(e); + } catch (IOException e) { + throw new ServiceException(e); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetClusterNodeLabelsRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetClusterNodeLabelsRequestPBImpl.java new file mode 100644 index 0000000..052c408 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetClusterNodeLabelsRequestPBImpl.java @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest; + +import com.google.protobuf.TextFormat; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetClusterNodeLabelsRequestProto; + +public class GetClusterNodeLabelsRequestPBImpl extends + GetClusterNodeLabelsRequest { + + GetClusterNodeLabelsRequestProto proto = GetClusterNodeLabelsRequestProto + .getDefaultInstance(); + GetClusterNodeLabelsRequestProto.Builder builder = null; + boolean viaProto = false; + + public GetClusterNodeLabelsRequestPBImpl() { + builder = GetClusterNodeLabelsRequestProto.newBuilder(); + } + + public GetClusterNodeLabelsRequestPBImpl(GetClusterNodeLabelsRequestProto proto) { + this.proto = proto; + viaProto = true; + } + + public GetClusterNodeLabelsRequestProto getProto() { + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) + return false; + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetClusterNodeLabelsResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetClusterNodeLabelsResponsePBImpl.java new file mode 100644 index 0000000..8d8331f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetClusterNodeLabelsResponsePBImpl.java @@ -0,0 +1,113 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + +import java.util.HashSet; +import java.util.Set; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetClusterNodeLabelsResponseProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetClusterNodeLabelsResponseProtoOrBuilder; + +public class GetClusterNodeLabelsResponsePBImpl extends + GetClusterNodeLabelsResponse { + Set labels; + GetClusterNodeLabelsResponseProto proto = GetClusterNodeLabelsResponseProto + .getDefaultInstance(); + GetClusterNodeLabelsResponseProto.Builder builder = null; + boolean viaProto = false; + + public GetClusterNodeLabelsResponsePBImpl() { + this.builder = GetClusterNodeLabelsResponseProto.newBuilder(); + } + + public GetClusterNodeLabelsResponsePBImpl( + GetClusterNodeLabelsResponseProto proto) { + this.proto = proto; + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = GetClusterNodeLabelsResponseProto.newBuilder(proto); + } + viaProto = false; + } + + private void mergeLocalToBuilder() { + if (this.labels != null && !this.labels.isEmpty()) { + builder.clearNodeLabels(); + builder.addAllNodeLabels(this.labels); + } + } + + private void mergeLocalToProto() { + if (viaProto) + maybeInitBuilder(); + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + public GetClusterNodeLabelsResponseProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void initNodeLabels() { + if (this.labels != null) { + return; + } + GetClusterNodeLabelsResponseProtoOrBuilder p = viaProto ? proto : builder; + this.labels = new HashSet(); + this.labels.addAll(p.getNodeLabelsList()); + } + + @Override + public void setNodeLabels(Set labels) { + maybeInitBuilder(); + if (labels == null || labels.isEmpty()) { + builder.clearNodeLabels(); + } + this.labels = labels; + } + + @Override + public Set getNodeLabels() { + initNodeLabels(); + return this.labels; + } + + @Override + public int hashCode() { + assert false : "hashCode not designed"; + return 0; + } + + @Override + public boolean equals(Object other) { + if (other == null) + return false; + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetNodesToLabelsRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetNodesToLabelsRequestPBImpl.java new file mode 100644 index 0000000..6781a36 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetNodesToLabelsRequestPBImpl.java @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + +import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsRequest; + +import com.google.protobuf.TextFormat; +import org.apache.hadoop.yarn.proto.YarnServiceProtos; + +public class GetNodesToLabelsRequestPBImpl extends GetNodesToLabelsRequest { + + YarnServiceProtos.GetNodesToLabelsRequestProto proto = + YarnServiceProtos.GetNodesToLabelsRequestProto + .getDefaultInstance(); + YarnServiceProtos.GetNodesToLabelsRequestProto.Builder builder = null; + boolean viaProto = false; + + public GetNodesToLabelsRequestPBImpl() { + builder = YarnServiceProtos.GetNodesToLabelsRequestProto.newBuilder(); + } + + public GetNodesToLabelsRequestPBImpl( + YarnServiceProtos.GetNodesToLabelsRequestProto proto) { + this.proto = proto; + viaProto = true; + } + + public YarnServiceProtos.GetNodesToLabelsRequestProto getProto() { + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) + return false; + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetNodesToLabelsResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetNodesToLabelsResponsePBImpl.java new file mode 100644 index 0000000..3404830 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetNodesToLabelsResponsePBImpl.java @@ -0,0 +1,168 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; + +import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.impl.pb.NodeIdPBImpl; +import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto; +import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsResponse; + +import com.google.common.collect.Sets; +import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdToLabelsProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetNodesToLabelsResponseProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetNodesToLabelsResponseProtoOrBuilder; + +public class GetNodesToLabelsResponsePBImpl extends + GetNodesToLabelsResponse { + GetNodesToLabelsResponseProto proto = GetNodesToLabelsResponseProto + .getDefaultInstance(); + GetNodesToLabelsResponseProto.Builder builder = null; + boolean viaProto = false; + + private Map> nodeToLabels; + + public GetNodesToLabelsResponsePBImpl() { + this.builder = GetNodesToLabelsResponseProto.newBuilder(); + } + + public GetNodesToLabelsResponsePBImpl(GetNodesToLabelsResponseProto proto) { + this.proto = proto; + this.viaProto = true; + } + + private void initNodeToLabels() { + if (this.nodeToLabels != null) { + return; + } + GetNodesToLabelsResponseProtoOrBuilder p = viaProto ? proto : builder; + List list = p.getNodeToLabelsList(); + this.nodeToLabels = new HashMap>(); + + for (NodeIdToLabelsProto c : list) { + this.nodeToLabels.put(new NodeIdPBImpl(c.getNodeId()), + Sets.newHashSet(c.getNodeLabelsList())); + } + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = GetNodesToLabelsResponseProto.newBuilder(proto); + } + viaProto = false; + } + + private void addNodeToLabelsToProto() { + maybeInitBuilder(); + builder.clearNodeToLabels(); + if (nodeToLabels == null) { + return; + } + Iterable iterable = + new Iterable() { + @Override + public Iterator iterator() { + return new Iterator() { + + Iterator>> iter = nodeToLabels + .entrySet().iterator(); + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + + @Override + public NodeIdToLabelsProto next() { + Entry> now = iter.next(); + return NodeIdToLabelsProto.newBuilder() + .setNodeId(convertToProtoFormat(now.getKey())) + .addAllNodeLabels(now.getValue()).build(); + } + + @Override + public boolean hasNext() { + return iter.hasNext(); + } + }; + } + }; + builder.addAllNodeToLabels(iterable); + } + + private void mergeLocalToBuilder() { + if (this.nodeToLabels != null) { + addNodeToLabelsToProto(); + } + } + + private void mergeLocalToProto() { + if (viaProto) + maybeInitBuilder(); + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + public GetNodesToLabelsResponseProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public Map> getNodeToLabels() { + initNodeToLabels(); + return this.nodeToLabels; + } + + @Override + public void setNodeToLabels(Map> map) { + initNodeToLabels(); + nodeToLabels.clear(); + nodeToLabels.putAll(map); + } + + private NodeIdProto convertToProtoFormat(NodeId t) { + return ((NodeIdPBImpl)t).getProto(); + } + + @Override + public int hashCode() { + assert false : "hashCode not designed"; + return 0; + } + + @Override + public boolean equals(Object other) { + if (other == null) + return false; + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/ResourceManagerAdministrationProtocolPBClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/ResourceManagerAdministrationProtocolPBClientImpl.java index e4828e2..c2d813b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/ResourceManagerAdministrationProtocolPBClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/ResourceManagerAdministrationProtocolPBClientImpl.java @@ -30,10 +30,8 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.ipc.RPCUtil; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.AddToClusterNodeLabelsRequestProto; -import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.GetClusterNodeLabelsRequestProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.GetGroupsForUserRequestProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.GetGroupsForUserResponseProto; -import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.GetNodesToLabelsRequestProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshAdminAclsRequestProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshNodesRequestProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshQueuesRequestProto; @@ -47,10 +45,6 @@ import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocolPB; import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsResponse; -import org.apache.hadoop.yarn.server.api.protocolrecords.GetClusterNodeLabelsRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.GetClusterNodeLabelsResponse; -import org.apache.hadoop.yarn.server.api.protocolrecords.GetNodesToLabelsRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.GetNodesToLabelsResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesRequest; @@ -71,10 +65,6 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.AddToClusterNodeLabelsRequestPBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.AddToClusterNodeLabelsResponsePBImpl; -import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.GetClusterNodeLabelsRequestPBImpl; -import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.GetClusterNodeLabelsResponsePBImpl; -import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.GetNodesToLabelsRequestPBImpl; -import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.GetNodesToLabelsResponsePBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RefreshAdminAclsRequestPBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RefreshAdminAclsResponsePBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RefreshNodesRequestPBImpl; @@ -273,32 +263,4 @@ public ReplaceLabelsOnNodeResponse replaceLabelsOnNode( return null; } } - - @Override - public GetNodesToLabelsResponse getNodeToLabels(GetNodesToLabelsRequest request) - throws YarnException, IOException { - GetNodesToLabelsRequestProto requestProto = - ((GetNodesToLabelsRequestPBImpl) request).getProto(); - try { - return new GetNodesToLabelsResponsePBImpl(proxy.getNodeToLabels( - null, requestProto)); - } catch (ServiceException e) { - RPCUtil.unwrapAndThrowException(e); - return null; - } - } - - @Override - public GetClusterNodeLabelsResponse getClusterNodeLabels( - GetClusterNodeLabelsRequest request) throws YarnException, IOException { - GetClusterNodeLabelsRequestProto requestProto = - ((GetClusterNodeLabelsRequestPBImpl) request).getProto(); - try { - return new GetClusterNodeLabelsResponsePBImpl(proxy.getClusterNodeLabels( - null, requestProto)); - } catch (ServiceException e) { - RPCUtil.unwrapAndThrowException(e); - return null; - } - } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/ResourceManagerAdministrationProtocolPBServiceImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/ResourceManagerAdministrationProtocolPBServiceImpl.java index 7cfecad..0eaf581 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/ResourceManagerAdministrationProtocolPBServiceImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/ResourceManagerAdministrationProtocolPBServiceImpl.java @@ -24,12 +24,8 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.AddToClusterNodeLabelsRequestProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.AddToClusterNodeLabelsResponseProto; -import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.GetClusterNodeLabelsRequestProto; -import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.GetClusterNodeLabelsResponseProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.GetGroupsForUserRequestProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.GetGroupsForUserResponseProto; -import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.GetNodesToLabelsRequestProto; -import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.GetNodesToLabelsResponseProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshAdminAclsRequestProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshAdminAclsResponseProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshNodesRequestProto; @@ -51,8 +47,6 @@ import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocol; import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocolPB; import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsResponse; -import org.apache.hadoop.yarn.server.api.protocolrecords.GetClusterNodeLabelsResponse; -import org.apache.hadoop.yarn.server.api.protocolrecords.GetNodesToLabelsResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshQueuesResponse; @@ -64,10 +58,6 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.AddToClusterNodeLabelsRequestPBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.AddToClusterNodeLabelsResponsePBImpl; -import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.GetClusterNodeLabelsRequestPBImpl; -import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.GetClusterNodeLabelsResponsePBImpl; -import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.GetNodesToLabelsRequestPBImpl; -import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.GetNodesToLabelsResponsePBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RefreshAdminAclsRequestPBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RefreshAdminAclsResponsePBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RefreshNodesRequestPBImpl; @@ -278,37 +268,4 @@ public ReplaceLabelsOnNodeResponseProto replaceLabelsOnNodes( throw new ServiceException(e); } } - - @Override - public GetNodesToLabelsResponseProto getNodeToLabels( - RpcController controller, GetNodesToLabelsRequestProto proto) - throws ServiceException { - GetNodesToLabelsRequestPBImpl request = - new GetNodesToLabelsRequestPBImpl(proto); - try { - GetNodesToLabelsResponse response = real.getNodeToLabels(request); - return ((GetNodesToLabelsResponsePBImpl) response).getProto(); - } catch (YarnException e) { - throw new ServiceException(e); - } catch (IOException e) { - throw new ServiceException(e); - } - } - - @Override - public GetClusterNodeLabelsResponseProto getClusterNodeLabels( - RpcController controller, GetClusterNodeLabelsRequestProto proto) - throws ServiceException { - GetClusterNodeLabelsRequestPBImpl request = - new GetClusterNodeLabelsRequestPBImpl(proto); - try { - GetClusterNodeLabelsResponse response = - real.getClusterNodeLabels(request); - return ((GetClusterNodeLabelsResponsePBImpl) response).getProto(); - } catch (YarnException e) { - throw new ServiceException(e); - } catch (IOException e) { - throw new ServiceException(e); - } - } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/GetClusterNodeLabelsRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/GetClusterNodeLabelsRequestPBImpl.java deleted file mode 100644 index 9cb4206..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/GetClusterNodeLabelsRequestPBImpl.java +++ /dev/null @@ -1,68 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb; - -import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.GetClusterNodeLabelsRequestProto; -import org.apache.hadoop.yarn.server.api.protocolrecords.GetClusterNodeLabelsRequest; - -import com.google.protobuf.TextFormat; - -public class GetClusterNodeLabelsRequestPBImpl extends - GetClusterNodeLabelsRequest { - - GetClusterNodeLabelsRequestProto proto = GetClusterNodeLabelsRequestProto - .getDefaultInstance(); - GetClusterNodeLabelsRequestProto.Builder builder = null; - boolean viaProto = false; - - public GetClusterNodeLabelsRequestPBImpl() { - builder = GetClusterNodeLabelsRequestProto.newBuilder(); - } - - public GetClusterNodeLabelsRequestPBImpl(GetClusterNodeLabelsRequestProto proto) { - this.proto = proto; - viaProto = true; - } - - public GetClusterNodeLabelsRequestProto getProto() { - proto = viaProto ? proto : builder.build(); - viaProto = true; - return proto; - } - - @Override - public int hashCode() { - return getProto().hashCode(); - } - - @Override - public boolean equals(Object other) { - if (other == null) - return false; - if (other.getClass().isAssignableFrom(this.getClass())) { - return this.getProto().equals(this.getClass().cast(other).getProto()); - } - return false; - } - - @Override - public String toString() { - return TextFormat.shortDebugString(getProto()); - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/GetClusterNodeLabelsResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/GetClusterNodeLabelsResponsePBImpl.java deleted file mode 100644 index 5465774..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/GetClusterNodeLabelsResponsePBImpl.java +++ /dev/null @@ -1,114 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb; - -import java.util.HashSet; -import java.util.Set; - -import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.GetClusterNodeLabelsResponseProto; -import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.GetClusterNodeLabelsResponseProtoOrBuilder; -import org.apache.hadoop.yarn.server.api.protocolrecords.GetClusterNodeLabelsResponse; - -public class GetClusterNodeLabelsResponsePBImpl extends - GetClusterNodeLabelsResponse { - Set labels; - GetClusterNodeLabelsResponseProto proto = GetClusterNodeLabelsResponseProto - .getDefaultInstance(); - GetClusterNodeLabelsResponseProto.Builder builder = null; - boolean viaProto = false; - - public GetClusterNodeLabelsResponsePBImpl() { - this.builder = GetClusterNodeLabelsResponseProto.newBuilder(); - } - - public GetClusterNodeLabelsResponsePBImpl( - GetClusterNodeLabelsResponseProto proto) { - this.proto = proto; - viaProto = true; - } - - private void maybeInitBuilder() { - if (viaProto || builder == null) { - builder = GetClusterNodeLabelsResponseProto.newBuilder(proto); - } - viaProto = false; - } - - private void mergeLocalToBuilder() { - if (this.labels != null && !this.labels.isEmpty()) { - builder.clearNodeLabels(); - builder.addAllNodeLabels(this.labels); - } - } - - private void mergeLocalToProto() { - if (viaProto) - maybeInitBuilder(); - mergeLocalToBuilder(); - proto = builder.build(); - viaProto = true; - } - - public GetClusterNodeLabelsResponseProto getProto() { - mergeLocalToProto(); - proto = viaProto ? proto : builder.build(); - viaProto = true; - return proto; - } - - private void initNodeLabels() { - if (this.labels != null) { - return; - } - GetClusterNodeLabelsResponseProtoOrBuilder p = viaProto ? proto : builder; - this.labels = new HashSet(); - this.labels.addAll(p.getNodeLabelsList()); - } - - @Override - public void setNodeLabels(Set labels) { - maybeInitBuilder(); - if (labels == null || labels.isEmpty()) { - builder.clearNodeLabels(); - } - this.labels = labels; - } - - @Override - public Set getNodeLabels() { - initNodeLabels(); - return this.labels; - } - - @Override - public int hashCode() { - assert false : "hashCode not designed"; - return 0; - } - - @Override - public boolean equals(Object other) { - if (other == null) - return false; - if (other.getClass().isAssignableFrom(this.getClass())) { - return this.getProto().equals(this.getClass().cast(other).getProto()); - } - return false; - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/GetNodesToLabelsRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/GetNodesToLabelsRequestPBImpl.java deleted file mode 100644 index 9e232eb..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/GetNodesToLabelsRequestPBImpl.java +++ /dev/null @@ -1,67 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb; - -import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.GetNodesToLabelsRequestProto; -import org.apache.hadoop.yarn.server.api.protocolrecords.GetNodesToLabelsRequest; - -import com.google.protobuf.TextFormat; - -public class GetNodesToLabelsRequestPBImpl extends GetNodesToLabelsRequest { - - GetNodesToLabelsRequestProto proto = GetNodesToLabelsRequestProto - .getDefaultInstance(); - GetNodesToLabelsRequestProto.Builder builder = null; - boolean viaProto = false; - - public GetNodesToLabelsRequestPBImpl() { - builder = GetNodesToLabelsRequestProto.newBuilder(); - } - - public GetNodesToLabelsRequestPBImpl(GetNodesToLabelsRequestProto proto) { - this.proto = proto; - viaProto = true; - } - - public GetNodesToLabelsRequestProto getProto() { - proto = viaProto ? proto : builder.build(); - viaProto = true; - return proto; - } - - @Override - public int hashCode() { - return getProto().hashCode(); - } - - @Override - public boolean equals(Object other) { - if (other == null) - return false; - if (other.getClass().isAssignableFrom(this.getClass())) { - return this.getProto().equals(this.getClass().cast(other).getProto()); - } - return false; - } - - @Override - public String toString() { - return TextFormat.shortDebugString(getProto()); - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/GetNodesToLabelsResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/GetNodesToLabelsResponsePBImpl.java deleted file mode 100644 index ec4e474..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/GetNodesToLabelsResponsePBImpl.java +++ /dev/null @@ -1,168 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb; - -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; - -import org.apache.hadoop.yarn.api.records.NodeId; -import org.apache.hadoop.yarn.api.records.impl.pb.NodeIdPBImpl; -import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto; -import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.GetNodesToLabelsResponseProto; -import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.GetNodesToLabelsResponseProtoOrBuilder; -import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.NodeIdToLabelsProto; -import org.apache.hadoop.yarn.server.api.protocolrecords.GetNodesToLabelsResponse; - -import com.google.common.collect.Sets; - -public class GetNodesToLabelsResponsePBImpl extends - GetNodesToLabelsResponse { - GetNodesToLabelsResponseProto proto = GetNodesToLabelsResponseProto - .getDefaultInstance(); - GetNodesToLabelsResponseProto.Builder builder = null; - boolean viaProto = false; - - private Map> nodeToLabels; - - public GetNodesToLabelsResponsePBImpl() { - this.builder = GetNodesToLabelsResponseProto.newBuilder(); - } - - public GetNodesToLabelsResponsePBImpl(GetNodesToLabelsResponseProto proto) { - this.proto = proto; - this.viaProto = true; - } - - private void initNodeToLabels() { - if (this.nodeToLabels != null) { - return; - } - GetNodesToLabelsResponseProtoOrBuilder p = viaProto ? proto : builder; - List list = p.getNodeToLabelsList(); - this.nodeToLabels = new HashMap>(); - - for (NodeIdToLabelsProto c : list) { - this.nodeToLabels.put(new NodeIdPBImpl(c.getNodeId()), - Sets.newHashSet(c.getNodeLabelsList())); - } - } - - private void maybeInitBuilder() { - if (viaProto || builder == null) { - builder = GetNodesToLabelsResponseProto.newBuilder(proto); - } - viaProto = false; - } - - private void addNodeToLabelsToProto() { - maybeInitBuilder(); - builder.clearNodeToLabels(); - if (nodeToLabels == null) { - return; - } - Iterable iterable = - new Iterable() { - @Override - public Iterator iterator() { - return new Iterator() { - - Iterator>> iter = nodeToLabels - .entrySet().iterator(); - - @Override - public void remove() { - throw new UnsupportedOperationException(); - } - - @Override - public NodeIdToLabelsProto next() { - Entry> now = iter.next(); - return NodeIdToLabelsProto.newBuilder() - .setNodeId(convertToProtoFormat(now.getKey())) - .addAllNodeLabels(now.getValue()).build(); - } - - @Override - public boolean hasNext() { - return iter.hasNext(); - } - }; - } - }; - builder.addAllNodeToLabels(iterable); - } - - private void mergeLocalToBuilder() { - if (this.nodeToLabels != null) { - addNodeToLabelsToProto(); - } - } - - private void mergeLocalToProto() { - if (viaProto) - maybeInitBuilder(); - mergeLocalToBuilder(); - proto = builder.build(); - viaProto = true; - } - - public GetNodesToLabelsResponseProto getProto() { - mergeLocalToProto(); - proto = viaProto ? proto : builder.build(); - viaProto = true; - return proto; - } - - @Override - public Map> getNodeToLabels() { - initNodeToLabels(); - return this.nodeToLabels; - } - - @Override - public void setNodeToLabels(Map> map) { - initNodeToLabels(); - nodeToLabels.clear(); - nodeToLabels.putAll(map); - } - - private NodeIdProto convertToProtoFormat(NodeId t) { - return ((NodeIdPBImpl)t).getProto(); - } - - @Override - public int hashCode() { - assert false : "hashCode not designed"; - return 0; - } - - @Override - public boolean equals(Object other) { - if (other == null) - return false; - if (other.getClass().isAssignableFrom(this.getClass())) { - return this.getProto().equals(this.getClass().cast(other).getProto()); - } - return false; - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReplaceLabelsOnNodeRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReplaceLabelsOnNodeRequestPBImpl.java index 6cf8dcc..e296aaf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReplaceLabelsOnNodeRequestPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReplaceLabelsOnNodeRequestPBImpl.java @@ -28,7 +28,7 @@ import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.impl.pb.NodeIdPBImpl; import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto; -import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.NodeIdToLabelsProto; +import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdToLabelsProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.ReplaceLabelsOnNodeRequestProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.ReplaceLabelsOnNodeRequestProtoOrBuilder; import org.apache.hadoop.yarn.server.api.protocolrecords.ReplaceLabelsOnNodeRequest; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java index 7ec3e79..14109f9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java @@ -59,10 +59,10 @@ import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocol; import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsResponse; -import org.apache.hadoop.yarn.server.api.protocolrecords.GetClusterNodeLabelsRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.GetClusterNodeLabelsResponse; -import org.apache.hadoop.yarn.server.api.protocolrecords.GetNodesToLabelsRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.GetNodesToLabelsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesRequest; @@ -81,8 +81,8 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.ReplaceLabelsOnNodeResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceResponse; -import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.GetClusterNodeLabelsResponsePBImpl; -import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.GetNodesToLabelsResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodeLabelsResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetNodesToLabelsResponsePBImpl; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystem; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeResourceUpdateEvent; @@ -722,18 +722,4 @@ public ReplaceLabelsOnNodeResponse replaceLabelsOnNode( throw RPCUtil.getRemoteException(ioe); } } - - @Override - public GetNodesToLabelsResponse getNodeToLabels(GetNodesToLabelsRequest request) - throws YarnException, IOException { - return GetNodesToLabelsResponsePBImpl.newInstance(rmContext - .getNodeLabelManager().getNodeLabels()); - } - - @Override - public GetClusterNodeLabelsResponse getClusterNodeLabels(GetClusterNodeLabelsRequest request) - throws YarnException, IOException { - return GetClusterNodeLabelsResponsePBImpl.newInstance(rmContext.getNodeLabelManager() - .getClusterNodeLabels()); - } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index 12811d0..477a3d1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -60,6 +60,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportRequest; @@ -70,6 +72,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoRequest; @@ -116,6 +120,7 @@ import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.reservation.Plan; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationInputValidator; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystem; @@ -1187,6 +1192,25 @@ public ReservationDeleteResponse deleteReservation( return response; } + @Override + public GetNodesToLabelsResponse getNodeToLabels( + GetNodesToLabelsRequest request) throws YarnException, IOException { + RMNodeLabelsManager labelsMgr = rmContext.getNodeLabelManager(); + GetNodesToLabelsResponse response = + GetNodesToLabelsResponse.newInstance(labelsMgr.getNodeLabels()); + return response; + } + + @Override + public GetClusterNodeLabelsResponse getClusterNodeLabels( + GetClusterNodeLabelsRequest request) throws YarnException, IOException { + RMNodeLabelsManager labelsMgr = rmContext.getNodeLabelManager(); + GetClusterNodeLabelsResponse response = + GetClusterNodeLabelsResponse.newInstance( + labelsMgr.getClusterNodeLabels()); + return response; + } + private void checkReservationSytem(String auditConstant) throws YarnException { // Check if reservation is enabled if (reservationSystem == null) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index e875cbf..642363e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -40,7 +40,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -63,7 +62,6 @@ import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; -import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;