diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index f7f82f8..681bf28 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -3326,6 +3326,11 @@ public static boolean isAclEnabled(Configuration conf) { public static final boolean DEFAULT_ROUTER_WEBAPP_PARTIAL_RESULTS_ENABLED = false; + public static final String ROUTER_CLIENTRM_PARTIAL_RESULTS_ENABLED = + ROUTER_WEBAPP_PREFIX + "partial-result.enabled"; + public static final boolean DEFAULT_ROUTER_CLIENTRM_PARTIAL_RESULTS_ENABLED = + false; + //////////////////////////////// // Other Configs //////////////////////////////// diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/uam/UnmanagedApplicationManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/uam/UnmanagedApplicationManager.java index 10985e0..6b81100 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/uam/UnmanagedApplicationManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/uam/UnmanagedApplicationManager.java @@ -86,6 +86,7 @@ LoggerFactory.getLogger(UnmanagedApplicationManager.class); private static final long AM_STATE_WAIT_TIMEOUT_MS = 10000; public static final String APP_NAME = "UnmanagedAM"; + public static final String PARTIAL_REPORT = "Partial Report "; private static final String DEFAULT_QUEUE_CONFIG = "uam.default.queue.name"; private BlockingQueue requestQueue; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java index 23cd3e2..9fb77f3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java @@ -531,8 +531,20 @@ public GetApplicationsResponse getApplications(GetApplicationsRequest request) throws YarnException, IOException { validateRunning(); - - return GetApplicationsResponse.newInstance(null); + List reports = new ArrayList<>(); + for (ApplicationId applicationId : applicationMap) { + GetApplicationReportResponse response = + Records.newRecord(GetApplicationReportResponse.class); + ApplicationReport report = Records.newRecord(ApplicationReport.class); + report.setYarnApplicationState(YarnApplicationState.ACCEPTED); + report.setApplicationId(applicationId); + report.setCurrentApplicationAttemptId( + ApplicationAttemptId.newInstance(applicationId, 1)); + report.setAMRMToken(Token.newInstance(new byte[0], "", new byte[0], "")); + response.setApplicationReport(report); + reports.add(report); + } + return GetApplicationsResponse.newInstance(reports); } @Override @@ -567,8 +579,8 @@ public GetDelegationTokenResponse getDelegationToken( GetDelegationTokenRequest request) throws YarnException, IOException { validateRunning(); - - return GetDelegationTokenResponse.newInstance(null); + Token token = Token.newInstance(new byte[0], "", new byte[0], ""); + return GetDelegationTokenResponse.newInstance(token); } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterMetrics.java index 6d75471..3edaac6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterMetrics.java @@ -51,6 +51,8 @@ private MutableGaugeInt numAppsFailedRetrieved; @Metric("# of multiple applications reports failed to be retrieved") private MutableGaugeInt numMultipleAppsFailedRetrieved; + @Metric("# of delegation token failed to obtain") + private MutableGaugeInt numDelegationTokenFailedObtain; // Aggregate metrics are shared, and don't have to be looked up per call @Metric("Total number of successful Submitted apps and latency(ms)") @@ -64,6 +66,9 @@ @Metric("Total number of successful Retrieved multiple apps reports and " + "latency(ms)") private MutableRate totalSucceededMultipleAppsRetrieved; + @Metric("Total number of successful Obtain delegation token and " + + "latency(ms)") + private MutableRate totalSucceededDelegationTokenObtain; /** * Provide quantile counters for all latencies. @@ -73,6 +78,7 @@ private MutableQuantiles killApplicationLatency; private MutableQuantiles getApplicationReportLatency; private MutableQuantiles getApplicationsReportLatency; + private MutableQuantiles obtainDelegationTokenLatency; private static volatile RouterMetrics INSTANCE = null; private static MetricsRegistry registry; @@ -92,6 +98,9 @@ private RouterMetrics() { getApplicationsReportLatency = registry.newQuantiles("getApplicationsReportLatency", "latency of get applications report", "ops", "latency", 10); + obtainDelegationTokenLatency = + registry.newQuantiles("obtainDelegationTokenLatency", + "latency of get delegation token", "ops", "latency", 10); } public static RouterMetrics getMetrics() { @@ -139,6 +148,11 @@ public long getNumSucceededMultipleAppsRetrieved() { } @VisibleForTesting + public long getNumSucceededDelegtationTokenObtain() { + return totalSucceededDelegationTokenObtain.lastStat().numSamples(); + } + + @VisibleForTesting public double getLatencySucceededAppsCreated() { return totalSucceededAppsCreated.lastStat().mean(); } @@ -164,6 +178,11 @@ public double getLatencySucceededMultipleGetAppReport() { } @VisibleForTesting + public double getLatencySucceededDelegationToken() { + return totalSucceededDelegationTokenObtain.lastStat().mean(); + } + + @VisibleForTesting public int getAppsFailedCreated() { return numAppsFailedCreated.value(); } @@ -188,6 +207,11 @@ public int getMultipleAppsFailedRetrieved() { return numMultipleAppsFailedRetrieved.value(); } + @VisibleForTesting + public int getDelegationTokenFailedObtain() { + return numDelegationTokenFailedObtain.value(); + } + public void succeededAppsCreated(long duration) { totalSucceededAppsCreated.add(duration); getNewApplicationLatency.add(duration); @@ -213,6 +237,11 @@ public void succeededMultipleAppsRetrieved(long duration) { getApplicationsReportLatency.add(duration); } + public void succeededDelegationTokenObtain(long duration) { + totalSucceededDelegationTokenObtain.add(duration); + obtainDelegationTokenLatency.add(duration); + } + public void incrAppsFailedCreated() { numAppsFailedCreated.incr(); } @@ -233,4 +262,8 @@ public void incrMultipleAppsFailedRetrieved() { numMultipleAppsFailedRetrieved.incr(); } + public void incrDelegtationTokenFailedObtain() { + numDelegationTokenFailedObtain.incr(); + } + } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java index 07eaf97..2625150 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java @@ -23,10 +23,17 @@ import java.util.List; import java.util.Map; import java.util.Random; +import java.util.concurrent.Callable; +import java.util.concurrent.CompletionService; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorCompletionService; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.commons.lang.NotImplementedException; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.util.concurrent.HadoopExecutors; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse; @@ -93,9 +100,12 @@ import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest; import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsResponse; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.factories.RecordFactory; +import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.federation.failover.FederationProxyProviderUtil; import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyUtils; import org.apache.hadoop.yarn.server.federation.policies.RouterPolicyFacade; @@ -141,6 +151,14 @@ private RouterPolicyFacade policyFacade; private RouterMetrics routerMetrics; private final Clock clock = new MonotonicClock(); + private final RecordFactory recordFactory = + RecordFactoryProvider.getRecordFactory(null); + + /** + * Thread pool used for asynchronous operations. + */ + private ExecutorService threadpool; + private boolean returnPartialReport; @Override public void init(String userName) { @@ -165,6 +183,15 @@ public void init(String userName) { clientRMProxies = new ConcurrentHashMap(); routerMetrics = RouterMetrics.getMetrics(); + + threadpool = HadoopExecutors.newCachedThreadPool( + new ThreadFactoryBuilder() + .setNameFormat("FederationClientInterceptor #%d") + .build()); + + returnPartialReport = conf.getBoolean( + YarnConfiguration.ROUTER_CLIENTRM_PARTIAL_RESULTS_ENABLED, + YarnConfiguration.DEFAULT_ROUTER_CLIENTRM_PARTIAL_RESULTS_ENABLED); } @Override @@ -564,7 +591,79 @@ public GetApplicationReportResponse getApplicationReport( @Override public GetApplicationsResponse getApplications(GetApplicationsRequest request) throws YarnException, IOException { - throw new NotImplementedException(); + + long startTime = clock.getTime(); + if (request == null) { + routerMetrics.incrAppsFailedRetrieved(); + RouterServerUtil.logAndThrowException( + "Missing getApplications request or information.", + null); + } + Map subClustersActive = null; + try { + subClustersActive = federationFacade.getSubClusters(true); + } catch (YarnException e) { + routerMetrics.incrMultipleAppsFailedRetrieved(); + RouterServerUtil + .logAndThrowException("Read FederationStateStore failed.", e); + } + + // Send the requests in parallel + CompletionService compSvc = + new ExecutorCompletionService<>(this.threadpool); + + for (final SubClusterId subClusterId : subClustersActive.keySet()) { + compSvc.submit(new Callable() { + @Override + public GetApplicationsResponse call() throws YarnException { + GetApplicationsResponse reps = null; + try { + ApplicationClientProtocol clientRMProxy = + getClientRMProxyForSubCluster(subClusterId); + reps = clientRMProxy.getApplications(request); + if (reps == null) { + routerMetrics.incrMultipleAppsFailedRetrieved(); + LOG.error("Subcluster {} failed to getApplications.", + subClusterId); + return null; + } + + } catch (Exception e) { + routerMetrics.incrMultipleAppsFailedRetrieved(); + LOG.error("Unable to getApplications.", e); + RouterServerUtil + .logAndThrowException("Unable to getApplications.", e); + } + return reps; + } + }); + } + + List apps = new ArrayList<>(); + // Collect all the responses in parallel + for (int i = 0; i < subClustersActive.size(); i++) { + try { + Future future = compSvc.take(); + GetApplicationsResponse response = future.get(); + + long stopTime = clock.getTime(); + routerMetrics.succeededMultipleAppsRetrieved(stopTime - startTime); + + if (response != null) { + apps.addAll(response.getApplicationList()); + } + } catch (Throwable e) { + routerMetrics.incrMultipleAppsFailedRetrieved(); + LOG.warn("Failed to get application report.", e); + } + } + + if (apps.isEmpty()) { + return null; + } + // Merge all the application report got from all the available YARN RMs + return RouterClientRMServiceUtils.mergeAppsReportInfo(apps, + returnPartialReport); } @Override @@ -674,7 +773,35 @@ public GetContainersResponse getContainers(GetContainersRequest request) @Override public GetDelegationTokenResponse getDelegationToken( GetDelegationTokenRequest request) throws YarnException, IOException { - throw new NotImplementedException(); + long startTime = clock.getTime(); + Map subClustersActive = + federationFacade.getSubClusters(true); + for (int i = 0; i < numSubmitRetries; ++i) { + SubClusterId subClusterId = getRandomActiveSubCluster(subClustersActive); + LOG.info("getDelegationToken try #{} on SubCluster {}", i, subClusterId); + ApplicationClientProtocol clientRMProxy = + getClientRMProxyForSubCluster(subClusterId); + GetDelegationTokenResponse response = null; + try { + response = clientRMProxy.getDelegationToken(request); + } catch (Exception e) { + LOG.warn("Unable to getDelegationToken in SubCluster {}", + subClusterId.getId(), e); + } + if (response != null) { + long stopTime = clock.getTime(); + routerMetrics.succeededDelegationTokenObtain(stopTime - startTime); + return response; + } else { + // Empty response from the ResourceManager. + // Blacklist this subcluster for this request. + subClustersActive.remove(subClusterId); + } + } + routerMetrics.incrDelegtationTokenFailedObtain(); + String errMsg = "Fail to getDelegationToken."; + LOG.error(errMsg); + throw new YarnException(errMsg); } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMServiceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMServiceUtils.java new file mode 100644 index 0000000..fa65f4a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMServiceUtils.java @@ -0,0 +1,219 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.router.clientrm; + +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; +import org.apache.hadoop.yarn.server.uam.UnmanagedApplicationManager; +import org.apache.hadoop.yarn.util.resource.Resources; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * The Router ClientRMService util class. + */ +public final class RouterClientRMServiceUtils { + + /** Disable constructor. */ + private RouterClientRMServiceUtils() { + } + /** + * Merges a list of ApplicationReport grouping by ApplicationId. + * Our current policy is to merge the application reports from the reacheable + * SubClusters. Via configuration parameter, we decide whether to return + * applications for which the primary AM is missing or to omit them. + * + * @param appsReport a list of AppInfo to merge + * @param returnPartialResult if the merge AppsInfo should contain partial + * result or not + * @return the merged GetApplicationsResponse + */ + public static GetApplicationsResponse mergeAppsReportInfo( + List appsReport, boolean returnPartialResult) { + + Map federationAM = + new HashMap(); + Map federationUAMSum = + new HashMap(); + for (ApplicationReport a : appsReport) { + // Check if this AppInfo is an AM + if (a.getName() != null && + !(a.getName().startsWith(UnmanagedApplicationManager.APP_NAME))) { + // Insert in the list of AM + federationAM.put(a.getApplicationId(), a); + // Check if there are any UAM found before + if (federationUAMSum.containsKey(a.getApplicationId())) { + // Merge the current AM with the found UAM + mergeAMWithUAM(a, federationUAMSum.get(a.getApplicationId())); + // Remove the sum of the UAMs + federationUAMSum.remove(a.getApplicationId()); + } + // This AppInfo is an UAM + } else { + if (federationAM.containsKey(a.getApplicationId())) { + // Merge the current UAM with its own AM + mergeAMWithUAM(federationAM.get(a.getApplicationId()), a); + } else if (federationUAMSum.containsKey(a.getApplicationId())) { + // Merge the current UAM with its own UAM and update the list of UAM + federationUAMSum.put(a.getApplicationId(), + mergeUAMWithUAM(federationUAMSum.get(a.getApplicationId()), a)); + } else { + // Insert in the list of UAM + federationUAMSum.put(a.getApplicationId(), a); + } + } + } + + // Check the remaining UAMs are depending or not from federation + for (ApplicationReport a : federationUAMSum.values()) { + if (returnPartialResult || (a.getName() != null + && !(a.getName().startsWith(UnmanagedApplicationManager.APP_NAME) + || a.getName().startsWith( + UnmanagedApplicationManager.PARTIAL_REPORT)))) { + federationAM.put(a.getApplicationId(), a); + } + } + + GetApplicationsResponse result = + GetApplicationsResponse.newInstance( + new ArrayList(federationAM.values())); + + return result; + } + + private static ApplicationReport mergeUAMWithUAM(ApplicationReport uam1, + ApplicationReport uam2) { + // We pick the status of the first uam + ApplicationReport partialReport = ApplicationReport.newInstance( + uam1.getApplicationId(), + uam1.getCurrentApplicationAttemptId(), + uam1.getUser(), + uam1.getQueue(), + UnmanagedApplicationManager.PARTIAL_REPORT + uam1.getApplicationId(), + uam1.getHost(), + uam1.getRpcPort(), + uam1.getClientToAMToken(), + uam1.getYarnApplicationState(), + uam1.getDiagnostics(), + uam1.getTrackingUrl(), + uam1.getStartTime(), + uam1.getLaunchTime(), + uam1.getFinishTime(), + uam1.getFinalApplicationStatus(), + uam1.getApplicationResourceUsageReport(), + uam1.getOriginalTrackingUrl(), + uam1.getProgress(), + uam1.getApplicationType(), + uam1.getAMRMToken(), + uam1.getApplicationTags(), + uam1.isUnmanagedApp(), + uam1.getPriority(), + uam1.getAppNodeLabelExpression(), + uam1.getAmNodeLabelExpression() + ); + + // Merge the newly partial AM with UAM1 and then with UAM2 + mergeAMWithUAM(partialReport, uam2); + return partialReport; + } + + private static void mergeAMWithUAM(ApplicationReport am, + ApplicationReport uam) { + + ApplicationResourceUsageReport report1 = + am.getApplicationResourceUsageReport(); + ApplicationResourceUsageReport report2 = + uam.getApplicationResourceUsageReport(); + ApplicationResourceUsageReport usageReport = + am.getApplicationResourceUsageReport(); + + ApplicationResourceUsageReport.newInstance( + report1.getNumUsedContainers() + + report2.getNumUsedContainers(), + report1.getNumReservedContainers() + + report2.getNumReservedContainers(), + Resources.add(report1.getUsedResources(), + report2.getUsedResources()), + Resources.add(report1.getReservedResources(), + report2.getReservedResources()), + Resources.add(report1.getNeededResources(), + report2.getNeededResources()), + merge(report1.getResourceSecondsMap(), + report2.getResourceSecondsMap()), + report1.getQueueUsagePercentage(), + report1.getClusterUsagePercentage(), + merge(report1.getPreemptedResourceSecondsMap(), + report2.getPreemptedResourceSecondsMap()) + ); + + usageReport.setNumUsedContainers(report1.getNumUsedContainers() + + report2.getNumUsedContainers()); + usageReport.setReservedResources(Resources.add( + report1.getReservedResources(), + report2.getReservedResources())); + usageReport.setUsedResources(Resources.add( + report1.getUsedResources(), + report2.getUsedResources())); + usageReport.setNeededResources(Resources.add( + report1.getNeededResources(), + report2.getNeededResources())); + + + usageReport.setNumUsedContainers(report1.getNumUsedContainers() + + report2.getNumUsedContainers()); + usageReport.setNumReservedContainers(report1.getNumReservedContainers() + + report2.getNumReservedContainers()); + usageReport.setMemorySeconds(report1.getMemorySeconds() + + report2.getMemorySeconds()); + usageReport.setVcoreSeconds(report1.getVcoreSeconds() + + report2.getVcoreSeconds()); + + usageReport.setPreemptedVcoreSeconds(report1.getPreemptedVcoreSeconds() + + report2.getPreemptedVcoreSeconds()); + usageReport.setPreemptedMemorySeconds(report1.getPreemptedMemorySeconds() + + report2.getPreemptedMemorySeconds()); + usageReport.setResourceSecondsMap(merge(report1.getResourceSecondsMap(), + report2.getResourceSecondsMap())); + usageReport.setPreemptedResourceSecondsMap(merge( + report1.getPreemptedResourceSecondsMap(), + report2.getPreemptedResourceSecondsMap())); + + am.setApplicationResourceUsageReport(usageReport); + } + + private static Map merge(Map l1, + Map l2) { + Map result = new HashMap<>(); + result.putAll(l1); + + for (Map.Entry kv : l2.entrySet()) { + if (result.containsKey(kv.getKey())) { + result.put(kv.getKey(), result.get(kv.getKey()) + kv.getValue()); + } else { + result.put(kv.getKey(), kv.getValue()); + } + } + return result; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java index 40bdbd8..92cba17 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java @@ -69,8 +69,6 @@ private static final Logger LOG = LoggerFactory.getLogger(RouterWebServiceUtil.class.getName()); - private final static String PARTIAL_REPORT = "Partial Report "; - /** Disable constructor. */ private RouterWebServiceUtil() { } @@ -297,7 +295,8 @@ public static AppsInfo mergeAppsInfo(ArrayList appsInfo, for (AppInfo a : federationUAMSum.values()) { if (returnPartialResult || (a.getName() != null && !(a.getName().startsWith(UnmanagedApplicationManager.APP_NAME) - || a.getName().startsWith(PARTIAL_REPORT)))) { + || a.getName().startsWith( + UnmanagedApplicationManager.PARTIAL_REPORT)))) { federationAM.put(a.getAppId(), a); } } @@ -309,7 +308,8 @@ public static AppsInfo mergeAppsInfo(ArrayList appsInfo, private static AppInfo mergeUAMWithUAM(AppInfo uam1, AppInfo uam2) { AppInfo partialReport = new AppInfo(); partialReport.setAppId(uam1.getAppId()); - partialReport.setName(PARTIAL_REPORT + uam1.getAppId()); + partialReport.setName(UnmanagedApplicationManager.PARTIAL_REPORT + + uam1.getAppId()); // We pick the status of the first uam partialReport.setState(uam1.getState()); // Merge the newly partial AM with UAM1 and then with UAM2 diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouterMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouterMetrics.java index 4c18ace..725e099 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouterMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouterMetrics.java @@ -53,6 +53,8 @@ public static void init() { Assert.assertEquals(0, metrics.getAppsFailedKilled()); Assert.assertEquals(0, metrics.getAppsFailedRetrieved()); + Assert.assertEquals(0, metrics.getDelegationTokenFailedObtain()); + LOG.info("Test: aggregate metrics are updated correctly"); } @@ -235,6 +237,46 @@ public void testMulipleAppsReportFailed() { metrics.getMultipleAppsFailedRetrieved()); } + /** + * This test validates the correctness of the metric: Get Delegation Token + * successfully. + */ + @Test + public void testSucceededDelegationTokenObtaion() { + + long totalGoodBefore = metrics.getNumSucceededDelegtationTokenObtain(); + + goodSubCluster.getDelegationToken(100); + + Assert.assertEquals(totalGoodBefore + 1, + metrics.getNumSucceededDelegtationTokenObtain()); + Assert.assertEquals(100, + metrics.getLatencySucceededDelegationToken(), + 0); + + goodSubCluster.getDelegationToken(200); + + Assert.assertEquals(totalGoodBefore + 2, + metrics.getNumSucceededDelegtationTokenObtain()); + Assert.assertEquals(150, metrics.getLatencySucceededDelegationToken(), + 0); + } + + /** + * This test validates the correctness of the metric: Failed to retrieve + * Multiple Apps. + */ + @Test + public void testDelegationTokenFailedObtaion() { + + long totalBadbefore = metrics.getMultipleAppsFailedRetrieved(); + + badSubCluster.getApplicationsReport(); + + Assert.assertEquals(totalBadbefore + 1, + metrics.getMultipleAppsFailedRetrieved()); + } + // Records failures for all calls private class MockBadSubCluster { public void getNewApplication() { @@ -294,5 +336,11 @@ public void getApplicationsReport(long duration) { duration); metrics.succeededMultipleAppsRetrieved(duration); } + + public void getDelegationToken(long duration) { + LOG.info("Mocked: successful getDelegationToken call with duration {}", + duration); + metrics.succeededDelegationTokenObtain(duration); + } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestFederationClientInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestFederationClientInterceptor.java index 87dfc95..6ae5fbc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestFederationClientInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestFederationClientInterceptor.java @@ -24,6 +24,10 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest; @@ -351,6 +355,53 @@ public void testGetApplicationReport() } /** + * This test validates the correctness of GetApplications in case the + * application exists in the cluster. + */ + @Test + public void testGetApplicationsReport() + throws YarnException, IOException, InterruptedException { + System.out + .println("Test FederationClientInterceptor: Get Applications report"); + + ApplicationId appId = + ApplicationId.newInstance(System.currentTimeMillis(), 1); + ApplicationSubmissionContext context = ApplicationSubmissionContext + .newInstance(appId, "", "", null, null, false, false, -1, null, null); + + SubmitApplicationRequest request = + SubmitApplicationRequest.newInstance(context); + // Submit the application we want the report later + SubmitApplicationResponse response = interceptor.submitApplication(request); + + Assert.assertNotNull(response); + Assert.assertNotNull(stateStoreUtil.queryApplicationHomeSC(appId)); + + ApplicationId appId2 = + ApplicationId.newInstance(System.currentTimeMillis(), 2); + ApplicationSubmissionContext context2 = ApplicationSubmissionContext + .newInstance(appId2, "", "", null, null, false, false, -1, null, null); + + SubmitApplicationRequest request2 = + SubmitApplicationRequest.newInstance(context2); + // Submit the application we want the report later + SubmitApplicationResponse response2 = + interceptor.submitApplication(request2); + + Assert.assertNotNull(response2); + Assert.assertNotNull(stateStoreUtil.queryApplicationHomeSC(appId2)); + + GetApplicationsRequest requestGet = + GetApplicationsRequest.newInstance(); + + GetApplicationsResponse responseGet = + interceptor.getApplications(requestGet); + + Assert.assertNotNull(responseGet); + Assert.assertEquals(responseGet.getApplicationList().size(), 2); + } + + /** * This test validates the correctness of GetApplicationReport in case the * application does not exist in StateStore. */ @@ -400,4 +451,23 @@ public void testGetApplicationEmptyRequest() } } + /** + * This test validates the correctness of Get Delegation Token in case of + * empty request. + */ + @Test + public void testGetDelegationToken() throws IOException, YarnException { + System.out.println( + "Test FederationClientInterceptor: Get Delegation Token"); + + GetDelegationTokenResponse response = + interceptor.getDelegationToken(null); + Assert.assertNotNull(response); + + GetDelegationTokenRequest request = + GetDelegationTokenRequest.newInstance("x"); + + response = interceptor.getDelegationToken(request); + Assert.assertNotNull(response); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestRouterClientRMServiceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestRouterClientRMServiceUtils.java new file mode 100644 index 0000000..55e6e61 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestRouterClientRMServiceUtils.java @@ -0,0 +1,433 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.router.clientrm; + +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; +import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceInformation; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.server.uam.UnmanagedApplicationManager; +import org.junit.Assert; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Test class to validate RouterClientRMServiceUtils methods. + */ +public class TestRouterClientRMServiceUtils { + private static final Logger LOG = + LoggerFactory.getLogger(TestRouterClientRMServiceUtils.class); + + private static final ApplicationId APPID1 = ApplicationId.newInstance(1, 1); + private static final ApplicationId APPID2 = ApplicationId.newInstance(2, 1); + private static final ApplicationId APPID3 = ApplicationId.newInstance(3, 1); + private static final ApplicationId APPID4 = ApplicationId.newInstance(4, 1); + + private static final ApplicationAttemptId APP_ATTEMT1 = + ApplicationAttemptId.newInstance(APPID1, 1); + private static final ApplicationAttemptId APP_ATTEMT2 = + ApplicationAttemptId.newInstance(APPID2, 2); + private static final ApplicationAttemptId APP_ATTEMT3 = + ApplicationAttemptId.newInstance(APPID3, 3); + private static final ApplicationAttemptId APP_ATTEMT4 = + ApplicationAttemptId.newInstance(APPID4, 4); + + /** + * This test validates the correctness of + * RouterClientRMServiceUtils#mergeAppsReportInfo in case we want to merge 4 + * AMs. The expected result would be the same 4 AMs. + */ + @Test + public void testMerge4DifferentApps() { + + List apps = new ArrayList<>(); + + ApplicationReport app1 = createApp(APPID1, APP_ATTEMT1, + YarnApplicationState.RUNNING, + FinalApplicationStatus.UNDEFINED, + createDefaultAppUsageReport(), false); + apps.add(app1); + + ApplicationReport app2 = createApp(APPID2, APP_ATTEMT2, + YarnApplicationState.RUNNING, + FinalApplicationStatus.UNDEFINED, + createDefaultAppUsageReport(), false); + apps.add(app2); + + ApplicationReport app3 = createApp(APPID3, APP_ATTEMT3, + YarnApplicationState.RUNNING, + FinalApplicationStatus.UNDEFINED, + createDefaultAppUsageReport(), false); + apps.add(app3); + + ApplicationReport app4 = createApp(APPID4, APP_ATTEMT4, + YarnApplicationState.RUNNING, + FinalApplicationStatus.UNDEFINED, + createDefaultAppUsageReport(), false); + apps.add(app4); + + GetApplicationsResponse result = RouterClientRMServiceUtils + .mergeAppsReportInfo(apps, false); + Assert.assertNotNull(result); + Assert.assertEquals(4, result.getApplicationList().size()); + + ApplicationReport appReport1 = null; + ApplicationReport appReport2 = null; + ApplicationReport appReport3 = null; + ApplicationReport appReport4 = null; + + List appIds = new ArrayList(); + + for (ApplicationReport app : result.getApplicationList()) { + appIds.add(app.getApplicationId().toString()); + if (app.getApplicationId().equals(APPID1.toString())) { + appReport1 = app; + } + if (app.getApplicationId().equals(APPID2.toString())) { + appReport2 = app; + } + if (app.getApplicationId().equals(APPID3.toString())) { + appReport3 = app; + } + if (app.getApplicationId().equals(APPID4.toString())) { + appReport4 = app; + } + } + + Assert.assertTrue(appIds.contains(APPID1.toString())); + Assert.assertTrue(appIds.contains(APPID2.toString())); + Assert.assertTrue(appIds.contains(APPID3.toString())); + Assert.assertTrue(appIds.contains(APPID4.toString())); + } + + /** + * This test validates the correctness of + * RouterClientRMServiceUtils#mergeAppsReportInfo in case we want to merge 2 + * UAMs and their own AM. The status of the AM is FINISHED, so we check the + * correctness of the merging of the historical values. The expected result + * would be 1 report with the merged information. + */ + @Test + public void testMergeAppsFinished() { + List apps = new ArrayList<>(); + int value = 1000; + ApplicationReport app1 = createApp(APPID1, APP_ATTEMT1, + YarnApplicationState.FINISHED, + FinalApplicationStatus.ENDED, + createDefaultAppUsageReport(), false); + setAppInfo(app1, value); + apps.add(app1); + + ApplicationReport uam1 = createApp(APPID1, APP_ATTEMT1, + YarnApplicationState.FINISHED, + FinalApplicationStatus.ENDED, + createDefaultAppUsageReport(), true); + setAppInfo(uam1, value); + apps.add(uam1); + + ApplicationReport uam2 = createApp(APPID1, APP_ATTEMT1, + YarnApplicationState.FINISHED, + FinalApplicationStatus.ENDED, + createDefaultAppUsageReport(), true); + setAppInfo(uam2, value); + apps.add(uam2); + + GetApplicationsResponse result = RouterClientRMServiceUtils + .mergeAppsReportInfo(apps, false); + Assert.assertNotNull(result); + Assert.assertEquals(1, result.getApplicationList().size()); + + ApplicationReport report = result.getApplicationList().get(0); + ApplicationResourceUsageReport usage = + report.getApplicationResourceUsageReport(); + Assert.assertEquals(APPID1.toString(), + report.getApplicationId().toString()); + Assert.assertEquals(value * 3, usage.getMemorySeconds()); + Assert.assertTrue(Resource.newInstance(value * 3, value * 3) + .equals(usage.getNeededResources())); + Assert.assertTrue(Resource.newInstance(value * 3, value * 3) + .equals(usage.getReservedResources())); + Assert.assertTrue(Resource.newInstance(value * 3, value * 3) + .equals(usage.getUsedResources())); + Assert.assertEquals(value * 3, usage.getPreemptedMemorySeconds()); + Assert.assertEquals(value * 3, usage.getNumReservedContainers()); + Assert.assertEquals(value * 3, usage.getNumUsedContainers()); + Assert.assertEquals(value * 3, usage.getPreemptedMemorySeconds()); + Assert.assertEquals(value, usage.getQueueUsagePercentage(), + 0.000001); // Expect UsagePercentage equal to AM + Assert.assertEquals(value * 3, usage.getVcoreSeconds()); + + } + + /** + * This test validates the correctness of + * RouterClientRMServiceUtils#mergeAppsReportInfo in case we want to merge 2 + * UAMs and their own AM. The status of the AM is RUNNING, so we check the + * correctness of the merging of the runtime values. + * The expected result would be 1 report with the merged information. + */ + @Test + public void testMergeAppsRunning() { + List apps = new ArrayList<>(); + int value = 1000; + ApplicationReport app1 = createApp(APPID1, APP_ATTEMT1, + YarnApplicationState.RUNNING, + FinalApplicationStatus.UNDEFINED, + createDefaultAppUsageReport(), false); + setAppInfoRunning(app1, value); + apps.add(app1); + + ApplicationReport uam1 = createApp(APPID1, APP_ATTEMT1, + YarnApplicationState.RUNNING, + FinalApplicationStatus.UNDEFINED, + createDefaultAppUsageReport(), true); + setAppInfoRunning(uam1, value); + apps.add(uam1); + + ApplicationReport uam2 = createApp(APPID1, APP_ATTEMT1, + YarnApplicationState.RUNNING, + FinalApplicationStatus.UNDEFINED, + createDefaultAppUsageReport(), true); + setAppInfoRunning(uam2, value); + apps.add(uam2); + GetApplicationsResponse result = RouterClientRMServiceUtils + .mergeAppsReportInfo(apps, false); + Assert.assertNotNull(result); + Assert.assertEquals(1, result.getApplicationList().size()); + + ApplicationReport report = result.getApplicationList().get(0); + ApplicationResourceUsageReport usage = + report.getApplicationResourceUsageReport(); + Assert.assertEquals(APPID1.toString(), + report.getApplicationId().toString()); + Assert.assertEquals(value * 3, usage.getMemorySeconds()); + Assert.assertTrue(Resource.newInstance(value * 3, value * 3) + .equals(usage.getNeededResources())); + Assert.assertTrue(Resource.newInstance(value * 3, value * 3) + .equals(usage.getReservedResources())); + Assert.assertTrue(Resource.newInstance(value * 3, value * 3) + .equals(usage.getUsedResources())); + Assert.assertEquals(value * 3, usage.getPreemptedMemorySeconds()); + Assert.assertEquals(value * 3, usage.getNumReservedContainers()); + Assert.assertEquals(value * 3, usage.getNumUsedContainers()); + Assert.assertEquals(value * 3, usage.getPreemptedMemorySeconds()); + Assert.assertEquals(value, usage.getQueueUsagePercentage(), + 0.000001); // Expect UsagePercentage equal to AM + Assert.assertEquals(value * 3, usage.getVcoreSeconds()); + + Assert.assertEquals(2, + usage.getPreemptedResourceSecondsMap().size()); + Assert.assertEquals(2, + usage.getResourceSecondsMap().size()); + + Assert.assertEquals(value * 3, + usage.getPreemptedResourceSecondsMap() + .get(ResourceInformation.VCORES_URI).longValue()); + Assert.assertEquals(value * 3, + usage.getPreemptedResourceSecondsMap() + .get(ResourceInformation.MEMORY_URI).longValue()); + + Assert.assertEquals(value * 3, + usage.getResourceSecondsMap() + .get(ResourceInformation.VCORES_URI).longValue()); + Assert.assertEquals(value * 3, + usage.getResourceSecondsMap() + .get(ResourceInformation.MEMORY_URI).longValue()); + } + + /** + * This test validates the correctness of + * RouterClientRMServiceUtils#mergeAppsReportInfo in case we want to merge 2 + * UAMs without their own AM. The expected result would be an empty report or + * a partial report of the 2 UAMs depending on the selected policy. + */ + @Test + public void testMerge2UAM() { + List apps = new ArrayList<>(); + int value = 1000; + + ApplicationReport uam1 = createApp(APPID1, APP_ATTEMT1, + YarnApplicationState.RUNNING, + FinalApplicationStatus.UNDEFINED, + createDefaultAppUsageReport(), true); + setAppInfoRunning(uam1, value); + apps.add(uam1); + + ApplicationReport uam2 = createApp(APPID1, APP_ATTEMT1, + YarnApplicationState.RUNNING, + FinalApplicationStatus.UNDEFINED, + createDefaultAppUsageReport(), true); + setAppInfoRunning(uam2, value); + apps.add(uam2); + GetApplicationsResponse result = RouterClientRMServiceUtils + .mergeAppsReportInfo(apps, false); + Assert.assertNotNull(result); + Assert.assertEquals(0, result.getApplicationList().size()); + + // By enabling partial result, the expected result would be a partial report + // of the 2 UAMs + GetApplicationsResponse result2 = RouterClientRMServiceUtils + .mergeAppsReportInfo(apps, true); + Assert.assertEquals(1, result2.getApplicationList().size()); + Assert.assertEquals(YarnApplicationState.RUNNING, + result2.getApplicationList().get(0).getYarnApplicationState()); + } + + /** + * This test validates the correctness of + * RouterClientRMServiceUtils#mergeAppsReportInfo in case we want to merge 1 + * UAM that does not depend on Federation. The excepted result would be the + * same app report. + */ + @Test + public void testMergeUAM() { + List apps = new ArrayList<>(); + int value = 1000; + ApplicationReport app1 = createApp(APPID1, APP_ATTEMT1, + YarnApplicationState.RUNNING, + FinalApplicationStatus.UNDEFINED, + createDefaultAppUsageReport(), false); + setAppInfoRunning(app1, value); + apps.add(app1); + + // in this case the result does not change if we enable partial result + GetApplicationsResponse result = RouterClientRMServiceUtils + .mergeAppsReportInfo(apps, false); + Assert.assertNotNull(result); + Assert.assertEquals(1, result.getApplicationList().size()); + + } + + private ApplicationReport createApp(ApplicationId applicationId, + ApplicationAttemptId applicationAttemptId, + YarnApplicationState state, + FinalApplicationStatus finalStatus, + ApplicationResourceUsageReport report, + boolean isUAM) { + String appName = applicationId.toString(); + if (isUAM) { + appName = UnmanagedApplicationManager.APP_NAME + appName; + } + return ApplicationReport.newInstance( + applicationId, + applicationAttemptId, + "user", + "queue", + appName, + "host", + 50320, + null, + state, + "diagnostics", + "url:///", + System.currentTimeMillis(), + System.currentTimeMillis(), + System.currentTimeMillis(), + finalStatus, + report, + "origTrackingUrl", + 0, + "YARN", + null, + null, + true, + Priority.UNDEFINED, + "label", + "amLabel" + + ); + } + + private ApplicationResourceUsageReport createDefaultAppUsageReport() { + return ApplicationResourceUsageReport.newInstance(1, + 1, + Resource.newInstance(1, 1), + Resource.newInstance(1, 1), + Resource.newInstance(1, 1), + new HashMap(), + 0.1f, + 0.1f, + new HashMap() + ); + } + + private void setAppInfo(ApplicationReport am, int value) { + ApplicationResourceUsageReport report = + am.getApplicationResourceUsageReport(); + + report.setClusterUsagePercentage(value); + report.setMemorySeconds(value); + report.setNeededResources(Resource.newInstance(value, value)); + report.setPreemptedMemorySeconds(value); + report.setNumReservedContainers(value); + report.setNumUsedContainers(value); + report.setPreemptedVcoreSeconds(value); + report.setQueueUsagePercentage(value); + report.setReservedResources(Resource.newInstance(value, value)); + report.setUsedResources(Resource.newInstance(value, value)); + report.setVcoreSeconds(value); + am.setApplicationResourceUsageReport(report); + } + + + private void setAppInfoRunning(ApplicationReport am, int value) { + ApplicationResourceUsageReport report = + am.getApplicationResourceUsageReport(); + + Map resourceSecondsMap = new HashMap<>(); + Map preemtedResourceSecondsMap = new HashMap<>(); + resourceSecondsMap.put(ResourceInformation.VCORES_URI, + Long.valueOf(value)); + resourceSecondsMap.put(ResourceInformation.MEMORY_URI, + Long.valueOf(value)); + + preemtedResourceSecondsMap.put(ResourceInformation.MEMORY_URI, + Long.valueOf(value)); + preemtedResourceSecondsMap.put(ResourceInformation.VCORES_URI, + Long.valueOf(value)); + + report.setClusterUsagePercentage(value); + report.setMemorySeconds(value); + report.setNeededResources(Resource.newInstance(value, value)); + report.setPreemptedMemorySeconds(value); + report.setNumReservedContainers(value); + report.setNumUsedContainers(value); + report.setPreemptedVcoreSeconds(value); + report.setQueueUsagePercentage(value); + report.setReservedResources(Resource.newInstance(value, value)); + report.setUsedResources(Resource.newInstance(value, value)); + report.setVcoreSeconds(value); + report.setPreemptedResourceSecondsMap(preemtedResourceSecondsMap); + report.setResourceSecondsMap(resourceSecondsMap); + + am.setApplicationResourceUsageReport(report); + } +}