Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterMetrics.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterMetrics.java --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterMetrics.java (revision e9fc0e18002dafa1cc65d586aae077d332426338) +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterMetrics.java (revision d59960685664db87f92bdfdb947fc92ec42003ae) @@ -56,6 +56,9 @@ @Metric("# of getClusterMetrics failed to be retrieved") private MutableGaugeInt numGetClusterMetricsFailedRetrieved; + @Metric("# of getClusterNodes failed to be retrieved") + private MutableGaugeInt numGetNodeToLabelsFailedRetrieved; + // Aggregate metrics are shared, and don't have to be looked up per call @Metric("Total number of successful Submitted apps and latency(ms)") private MutableRate totalSucceededAppsSubmitted; @@ -75,6 +78,10 @@ + "latency(ms)") private MutableRate totalSucceededGetClusterMetricsRetrieved; + @Metric("Total number of successful Retrieved GetNodeToLabels and " + + "latency(ms)") + private MutableRate totalSucceededGetNodeToLabelsRetrieved; + /** * Provide quantile counters for all latencies. @@ -87,6 +94,8 @@ private MutableQuantiles getApplicationAttemptReportLatency; private MutableQuantiles getClusterMetricsLatency; + private MutableQuantiles getNodeToLabelsLatency; + private static volatile RouterMetrics INSTANCE = null; private static MetricsRegistry registry; @@ -112,6 +121,10 @@ getClusterMetricsLatency = registry.newQuantiles("getClusterMetricsLatency", "latency of get cluster metrics", "ops", "latency", 10); + + getNodeToLabelsLatency = + registry.newQuantiles("getNodeToLabelsLatency", + "latency of get node labels", "ops", "latency", 10); } public static RouterMetrics getMetrics() { @@ -168,6 +181,11 @@ return totalSucceededGetClusterMetricsRetrieved.lastStat().numSamples(); } + @VisibleForTesting + public long getNumSucceededGetNodeToLabelsRetrieved(){ + return totalSucceededGetNodeToLabelsRetrieved.lastStat().numSamples(); + } + @VisibleForTesting public double getLatencySucceededAppsCreated() { return totalSucceededAppsCreated.lastStat().mean(); @@ -203,6 +221,11 @@ return totalSucceededGetClusterMetricsRetrieved.lastStat().mean(); } + @VisibleForTesting + public double getLatencySucceededGetNodeToLabelsRetrieved() { + return totalSucceededGetNodeToLabelsRetrieved.lastStat().mean(); + } + @VisibleForTesting public int getAppsFailedCreated() { return numAppsFailedCreated.value(); @@ -238,6 +261,11 @@ return numGetClusterMetricsFailedRetrieved.value(); } + @VisibleForTesting + public int getNodeToLabelsFailedRetrieved() { + return numGetNodeToLabelsFailedRetrieved.value(); + } + public void succeededAppsCreated(long duration) { totalSucceededAppsCreated.add(duration); getNewApplicationLatency.add(duration); @@ -273,6 +301,11 @@ getClusterMetricsLatency.add(duration); } + public void succeededGetNodeToLabelsRetrieved(long duration) { + totalSucceededGetNodeToLabelsRetrieved.add(duration); + getNodeToLabelsLatency.add(duration); + } + public void incrAppsFailedCreated() { numAppsFailedCreated.incr(); } @@ -301,4 +334,8 @@ numGetClusterMetricsFailedRetrieved.incr(); } + public void incrGetNodeToLabelsFailedRetrieved() { + numGetNodeToLabelsFailedRetrieved.incr(); + } + } Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java (revision e9fc0e18002dafa1cc65d586aae077d332426338) +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java (revision d59960685664db87f92bdfdb947fc92ec42003ae) @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.router.clientrm; +import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; import java.io.IOException; import java.lang.reflect.Method; @@ -845,8 +846,32 @@ @Override public GetNodesToLabelsResponse getNodeToLabels( - GetNodesToLabelsRequest request) throws YarnException, IOException { - throw new NotImplementedException("Code is not implemented"); + GetNodesToLabelsRequest request) throws YarnException, IOException { + if (request == null) { + routerMetrics.incrGetNodeToLabelsFailedRetrieved(); + RouterServerUtil.logAndThrowException( + "Missing getNodeToLabels request.", null); + } + long startTime = clock.getTime(); + Map subClusters = + federationFacade.getSubClusters(true); + Map clusterNodeLables = Maps.newHashMap(); + for (Map.Entry entry : subClusters.entrySet()) { + SubClusterId subClusterId = entry.getKey(); + ApplicationClientProtocol client = null; + try { + client = getClientRMProxyForSubCluster(subClusterId); + GetNodesToLabelsResponse response = client.getNodeToLabels(request); + clusterNodeLables.put(subClusterId, response); + } catch (Exception ex) { + routerMetrics.incrGetNodeToLabelsFailedRetrieved(); + LOG.error("Unable to get cluster node labels due to exception.", ex); + } + } + long stopTime = clock.getTime(); + routerMetrics.succeededGetNodeToLabelsRetrieved(stopTime - startTime); + // merge cluster nodes + return RouterYarnClientUtils.mergeNodesToLabelsResponse(clusterNodeLables.values()); } @Override Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterYarnClientUtils.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterYarnClientUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterYarnClientUtils.java --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterYarnClientUtils.java (revision e9fc0e18002dafa1cc65d586aae077d332426338) +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterYarnClientUtils.java (revision d59960685664db87f92bdfdb947fc92ec42003ae) @@ -20,14 +20,14 @@ import java.util.Collection; import java.util.HashMap; import java.util.Map; +import java.util.Set; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse; -import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.ApplicationReport; -import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; -import org.apache.hadoop.yarn.api.records.YarnClusterMetrics; +import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsResponse; +import org.apache.hadoop.yarn.api.records.*; import org.apache.hadoop.yarn.server.uam.UnmanagedApplicationManager; +import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.util.resource.Resources; /** @@ -194,4 +194,24 @@ return !(appName.startsWith(UnmanagedApplicationManager.APP_NAME) || appName.startsWith(PARTIAL_REPORT)); } + + /** + * Merges a list of GetNodesToLabelsResponse grouping by SubClusterId. + * + * @param responses a list of GetNodesToLabelsResponse to merge + * @return the merged GetNodesToLabelsResponse + */ + public static GetNodesToLabelsResponse mergeNodesToLabelsResponse( + Collection responses) { + GetNodesToLabelsResponse nodesToLabelsResponse = Records.newRecord( + GetNodesToLabelsResponse.class); + Map> nodesToLabelMap = new HashMap<>(); + for (GetNodesToLabelsResponse response : responses) { + if (response != null && response.getNodeToLabels() != null) { + nodesToLabelMap.putAll(response.getNodeToLabels()); + } + } + nodesToLabelsResponse.setNodeToLabels(nodesToLabelMap); + return nodesToLabelsResponse; + } } Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestFederationClientInterceptor.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestFederationClientInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestFederationClientInterceptor.java --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestFederationClientInterceptor.java (revision e9fc0e18002dafa1cc65d586aae077d332426338) +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestFederationClientInterceptor.java (revision d59960685664db87f92bdfdb947fc92ec42003ae) @@ -30,20 +30,7 @@ import org.apache.hadoop.test.LambdaTestUtils; import org.apache.hadoop.yarn.MockApps; -import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse; -import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; -import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; -import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse; -import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; -import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest; -import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse; -import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; -import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.*; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; @@ -641,4 +628,17 @@ Assert.assertNotNull(responseGet); Assert.assertTrue(responseGet.getApplicationList().isEmpty()); } + + @Test + public void testGetNodeToLabelsRequest() throws Exception { + LOG.info("Test FederationClientInterceptor : GetNodeToLabels request"); + // null request + LambdaTestUtils.intercept(YarnException.class, + "Missing getNodeToLabels request.", () -> interceptor.getNodeToLabels(null)); + + // normal request. + GetNodesToLabelsResponse response = + interceptor.getNodeToLabels(GetNodesToLabelsRequest.newInstance()); + Assert.assertEquals(0, response.getNodeToLabels().size()); + } }