diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index 55a3f0b..b5c5899 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -51,6 +51,7 @@ import org.apache.hadoop.security.authorize.PolicyProvider; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.util.StopWatch; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope; @@ -1294,9 +1295,12 @@ public ReservationSubmissionResponse submitReservation( AuditConstants.SUBMIT_RESERVATION_REQUEST, null); try { // Try to place the reservation using the agent - boolean result = - plan.getReservationAgent().createReservation(reservationId, user, - plan, request.getReservationDefinition()); + StopWatch stopWatch = new StopWatch(); + stopWatch.start(); + boolean result = plan.getReservationAgent().createReservation( + reservationId, user, plan, request.getReservationDefinition()); + plan.getReservationQueueMetrics() + .setAgentCreateReservationMetrics(stopWatch.now()); if (result) { // add the reservation id to valid ones maintained by reservation // system @@ -1335,9 +1339,12 @@ public ReservationUpdateResponse updateReservation( AuditConstants.UPDATE_RESERVATION_REQUEST, reservationId); // Try to update the reservation using default agent try { - boolean result = - plan.getReservationAgent().updateReservation(reservationId, user, - plan, request.getReservationDefinition()); + StopWatch stopWatch = new StopWatch(); + stopWatch.start(); + boolean result = plan.getReservationAgent().updateReservation( + reservationId, user, plan, request.getReservationDefinition()); + plan.getReservationQueueMetrics() + .setAgentUpdateReservationMetrics(stopWatch.now()); if (!result) { String errMsg = "Unable to update reservation: " + reservationId; RMAuditLogger.logFailure(user, @@ -1374,9 +1381,12 @@ public ReservationDeleteResponse deleteReservation( AuditConstants.DELETE_RESERVATION_REQUEST, reservationId); // Try to update the reservation using default agent try { - boolean result = - plan.getReservationAgent().deleteReservation(reservationId, user, - plan); + StopWatch stopWatch = new StopWatch(); + stopWatch.start(); + boolean result = plan.getReservationAgent() + .deleteReservation(reservationId, user, plan); + plan.getReservationQueueMetrics() + .setAgentDeleteReservationMetrics(stopWatch.now()); if (!result) { String errMsg = "Could not delete reservation: " + reservationId; RMAuditLogger.logFailure(user, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java index 5b8772c..669bc67 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java @@ -419,7 +419,8 @@ protected Plan initializePlan(String planQueueName) throws YarnException { Resource maxAllocation = getMaxAllocation(); ResourceCalculator rescCalc = getResourceCalculator(); Resource totCap = getPlanQueueCapacity(planQueueName); - Plan plan = new InMemoryPlan(getRootQueueMetrics(), adPolicy, + Plan plan = new InMemoryPlan(getRootQueueMetrics(), + getRootQueueReservationMetrics(), adPolicy, getAgent(planQueuePath), totCap, planStepSize, rescCalc, minAllocation, maxAllocation, planQueueName, getReplanner(planQueuePath), getReservationSchedulerConfiguration().getMoveOnExpiry(planQueuePath), @@ -515,4 +516,6 @@ public ReservationsACLsManager getReservationsACLsManager() { protected abstract ResourceCalculator getResourceCalculator(); protected abstract QueueMetrics getRootQueueMetrics(); + + protected abstract ReservationQueueMetrics getRootQueueReservationMetrics(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractSchedulerPlanFollower.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractSchedulerPlanFollower.java index 9b6a0b0..8f339e5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractSchedulerPlanFollower.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractSchedulerPlanFollower.java @@ -26,6 +26,7 @@ import java.util.List; import java.util.Set; +import org.apache.hadoop.util.StopWatch; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ReservationId; import org.apache.hadoop.yarn.api.records.Resource; @@ -76,6 +77,8 @@ public synchronized void synchronizePlan(Plan plan, boolean shouldReplan) { if (LOG.isDebugEnabled()) { LOG.debug("Running plan follower edit policy for plan: " + planQueueName); } + StopWatch stopWatch = new StopWatch(); + stopWatch.start(); // align with plan step long step = plan.getStep(); long now = clock.getTime(); @@ -118,8 +121,7 @@ public synchronized void synchronizePlan(Plan plan, boolean shouldReplan) { shouldResize = true; } } - // identify the reservations that have expired and new reservations that - // have to be activated + // identify expired as well as new reservations that have to be activated List resQueues = getChildReservationQueues(planQueue); Set expired = new HashSet(); for (Queue resQueue : resQueues) { @@ -215,10 +217,11 @@ public synchronized void synchronizePlan(Plan plan, boolean shouldReplan) { } catch (PlanningException e) { LOG.error("Exception in archiving completed reservations: ", e); } + planQueue.getReservationMetrics() + .setPlanFollowerSynchronizeMetrics(stopWatch.now()); LOG.info("Finished iteration of plan follower edit policy for plan: " + planQueueName); - // Extension: update plan with app states, - // useful to support smart replanning + // Extension: update plan with app states, to support smart replanning. } protected String getReservationIdFromQueueName(String resQueueName) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacityReservationSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacityReservationSystem.java index be0a708..a3832de 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacityReservationSystem.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacityReservationSystem.java @@ -86,6 +86,11 @@ protected QueueMetrics getRootQueueMetrics() { } @Override + protected ReservationQueueMetrics getRootQueueReservationMetrics() { + return capScheduler.getRootQueueReservationMetrics(); + } + + @Override protected String getPlanQueuePath(String planQueueName) { return capScheduler.getQueue(planQueueName).getQueuePath(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/FairReservationSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/FairReservationSystem.java index 611fca8..0cd450e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/FairReservationSystem.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/FairReservationSystem.java @@ -69,6 +69,11 @@ protected QueueMetrics getRootQueueMetrics() { } @Override + protected ReservationQueueMetrics getRootQueueReservationMetrics() { + return fairScheduler.getRootQueueReservationMetrics(); + } + + @Override protected Resource getMinAllocation() { return fairScheduler.getMinimumResourceCapability(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/InMemoryPlan.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/InMemoryPlan.java index 49e07aa..e3a231c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/InMemoryPlan.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/InMemoryPlan.java @@ -31,6 +31,7 @@ import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantReadWriteLock; +import org.apache.hadoop.util.StopWatch; import org.apache.hadoop.yarn.api.records.ReservationId; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -97,29 +98,34 @@ private Resource totalCapacity; - public InMemoryPlan(QueueMetrics queueMetrics, SharingPolicy policy, + private ReservationQueueMetrics reservationQueueMetrics; + + public InMemoryPlan(QueueMetrics queueMetrics, + ReservationQueueMetrics reservationQueueMetrics,SharingPolicy policy, ReservationAgent agent, Resource totalCapacity, long step, ResourceCalculator resCalc, Resource minAlloc, Resource maxAlloc, String queueName, Planner replanner, boolean getMoveOnExpiry, RMContext rmContext) { - this(queueMetrics, policy, agent, totalCapacity, step, resCalc, minAlloc, - maxAlloc, queueName, replanner, getMoveOnExpiry, + this(queueMetrics, reservationQueueMetrics,policy, agent, totalCapacity, + step, resCalc, minAlloc, maxAlloc, queueName, replanner, getMoveOnExpiry, YarnConfiguration.DEFAULT_RM_RESERVATION_SYSTEM_MAX_PERIODICITY, rmContext); } - public InMemoryPlan(QueueMetrics queueMetrics, SharingPolicy policy, + public InMemoryPlan(QueueMetrics queueMetrics, + ReservationQueueMetrics reservationQueueMetrics, SharingPolicy policy, ReservationAgent agent, Resource totalCapacity, long step, ResourceCalculator resCalc, Resource minAlloc, Resource maxAlloc, String queueName, Planner replanner, boolean getMoveOnExpiry, long maxPeriodicity, RMContext rmContext) { - this(queueMetrics, policy, agent, totalCapacity, step, resCalc, minAlloc, + this(queueMetrics, reservationQueueMetrics, policy, agent, totalCapacity, step, resCalc, minAlloc, maxAlloc, queueName, replanner, getMoveOnExpiry, maxPeriodicity, rmContext, new UTCClock()); } @SuppressWarnings("checkstyle:parameternumber") - public InMemoryPlan(QueueMetrics queueMetrics, SharingPolicy policy, + public InMemoryPlan(QueueMetrics queueMetrics, + ReservationQueueMetrics reservationQueueMetrics, SharingPolicy policy, ReservationAgent agent, Resource totalCapacity, long step, ResourceCalculator resCalc, Resource minAlloc, Resource maxAlloc, String queueName, Planner replanner, boolean getMoveOnExpiry, @@ -141,6 +147,7 @@ public InMemoryPlan(QueueMetrics queueMetrics, SharingPolicy policy, this.getMoveOnExpiry = getMoveOnExpiry; this.clock = clock; this.rmStateStore = rmContext.getStateStore(); + this.reservationQueueMetrics = reservationQueueMetrics; } @Override @@ -182,6 +189,11 @@ private void gcUserRLEResourceAllocation(String user, long period) { } } + @Override + public ReservationQueueMetrics getReservationQueueMetrics() { + return reservationQueueMetrics; + } + private void incrementAllocation(ReservationAllocation reservation) { assert (readWriteLock.isWriteLockedByCurrentThread()); Map allocationRequests = @@ -325,6 +337,8 @@ private void decrementAllocation(ReservationAllocation reservation) { @Override public boolean addReservation(ReservationAllocation reservation, boolean isRecovering) throws PlanningException { + StopWatch stopWatch = new StopWatch(); + stopWatch.start(); // Verify the allocation is memory based otherwise it is not supported InMemoryReservationAllocation inMemReservation = (InMemoryReservationAllocation) reservation; @@ -332,8 +346,11 @@ public boolean addReservation(ReservationAllocation reservation, String errMsg = "The specified Reservation with ID " + inMemReservation.getReservationId() + " is not mapped to any user"; LOG.error(errMsg); + reservationQueueMetrics.setPlanAddReservationMetrics(stopWatch.now(), + false); throw new IllegalArgumentException(errMsg); } + writeLock.lock(); try { if (reservationTable.containsKey(inMemReservation.getReservationId())) { @@ -364,6 +381,8 @@ public boolean addReservation(ReservationAllocation reservation, if (!reservations.add(inMemReservation)) { LOG.error("Unable to add reservation: {} to plan.", inMemReservation.getReservationId()); + reservationQueueMetrics.setPlanAddReservationMetrics(stopWatch.now(), + false); return false; } currentReservations.put(searchInterval, reservations); @@ -372,7 +391,13 @@ public boolean addReservation(ReservationAllocation reservation, incrementAllocation(inMemReservation); LOG.info("Successfully added reservation: {} to plan.", inMemReservation.getReservationId()); + reservationQueueMetrics.setPlanAddReservationMetrics(stopWatch.now(), + true); return true; + } catch (Exception e) { + reservationQueueMetrics.setPlanAddReservationMetrics(stopWatch.now(), + false); + throw e; } finally { writeLock.unlock(); } @@ -381,6 +406,8 @@ public boolean addReservation(ReservationAllocation reservation, @Override public boolean updateReservation(ReservationAllocation reservation) throws PlanningException { + StopWatch stopWatch = new StopWatch(); + stopWatch.start(); writeLock.lock(); boolean result = false; try { @@ -419,6 +446,8 @@ public boolean updateReservation(ReservationAllocation reservation) } } finally { writeLock.unlock(); + reservationQueueMetrics.setPlanUpdateReservationMetrics(stopWatch.now(), + result); } } @@ -456,6 +485,8 @@ private boolean removeReservation(ReservationAllocation reservation) { @Override public boolean deleteReservation(ReservationId reservationID) { + StopWatch stopWatch = new StopWatch(); + stopWatch.start(); writeLock.lock(); try { ReservationAllocation reservation = getReservationById(reservationID); @@ -465,7 +496,14 @@ public boolean deleteReservation(ReservationId reservationID) { LOG.error(errMsg); throw new IllegalArgumentException(errMsg); } - return removeReservation(reservation); + boolean result = removeReservation(reservation); + reservationQueueMetrics.setPlanDeleteReservationMetrics(stopWatch.now(), + result); + return result; + } catch (Exception e){ + reservationQueueMetrics.setPlanDeleteReservationMetrics(stopWatch.now(), + false); + throw e; } finally { writeLock.unlock(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/PlanContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/PlanContext.java index 117a627..b207505 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/PlanContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/PlanContext.java @@ -120,9 +120,18 @@ public QueueMetrics getQueueMetrics(); /** + * Return the {@link ReservationQueueMetrics} for for the queue in the + * {@link ResourceScheduler} corresponding to this plan. + * + * @return the {@link ReservationQueueMetrics} for the queue in the + * {@link ResourceScheduler} corresponding to this plan + */ + ReservationQueueMetrics getReservationQueueMetrics(); + + /** * Instructs the {@link PlanFollower} on what to do for applications * which are still running when the reservation is expiring (move-to-default - * vs kill) + * vs kill). * * @return true if remaining applications have to be killed, false if they * have to migrated diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationQueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationQueueMetrics.java new file mode 100644 index 0000000..0028b86 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationQueueMetrics.java @@ -0,0 +1,264 @@ +/******************************************************************************* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ + +package org.apache.hadoop.yarn.server.resourcemanager.reservation; + +import static org.apache.hadoop.metrics2.lib.Interns.info; + +import java.util.HashMap; +import java.util.Map; + +import com.google.common.base.Splitter; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.metrics2.MetricsInfo; +import org.apache.hadoop.metrics2.MetricsSystem; +import org.apache.hadoop.metrics2.annotation.Metric; +import org.apache.hadoop.metrics2.annotation.Metrics; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.apache.hadoop.metrics2.lib.MetricsRegistry; +import org.apache.hadoop.metrics2.lib.MutableCounterInt; +import org.apache.hadoop.metrics2.lib.MutableQuantiles; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue; + +/** + * {@link ReservationQueueMetrics} is used to collect metrics for the + * {@link ReservationSystem} and its constituent components. + */ +@InterfaceAudience.Private +@Metrics(context = "yarn") +public class ReservationQueueMetrics { + + static final Splitter Q_SPLITTER = + Splitter.on('.').omitEmptyStrings().trimResults(); + private static MetricsRegistry registry; + + /** + * Helper method to clear cache. + */ + @InterfaceAudience.Private + public synchronized static void clearReservationMetrics() { + RES_METRICS.clear(); + } + + /** + * Simple metrics cache to help prevent re-registrations. + */ + private static final Map RES_METRICS = + new HashMap<>(); + + /** + * Gets the instance of {@link ReservationQueueMetrics} for a particular + * queue. If an instance did not already exist, a new one will be created. The + * {@link MetricsSystem} used to register these metrics is the singleton + * {@link DefaultMetricsSystem#instance()}. + * + * @param queueName the name of the {@link Queue} that the + * {@link ReservationQueueMetrics} are being gathered for. + * @param parent the parent of the {@link Queue} that the + * {@link ReservationQueueMetrics} are being gathered for. + * @return the {@link ReservationQueueMetrics} for the queue represented + * by the parameter queueName. + */ + public synchronized static ReservationQueueMetrics forReservationQueue( + String queueName, Queue parent) { + return forReservationQueue(DefaultMetricsSystem.instance(), queueName, + parent); + } + + /** + * Gets the instance of {@link ReservationQueueMetrics} for a particular + * queue. If an instance did not already exist, a new one will be created. + * + * @param ms the {@link MetricsSystem} used to register the queue metrics. + * @param queueName the name of the {@link Queue} that the + * {@link ReservationQueueMetrics} are being gathered for. + * @param parent the parent of the {@link Queue} that the + * {@link ReservationQueueMetrics} are being gathered for. + * @return the {@link ReservationQueueMetrics} for the queue represented + * by the parameter queueName. + */ + public synchronized static ReservationQueueMetrics forReservationQueue( + MetricsSystem ms, String queueName, Queue parent) { + ReservationQueueMetrics metrics = RES_METRICS.get(queueName); + if (metrics == null) { + metrics = new ReservationQueueMetrics(queueName, parent); + metrics.initialize(); + + // Register with the MetricsSystems + if (ms != null) { + metrics = ms.register("ReservationQueue-" + sourceName(queueName) + .toString(), "Metrics for queue: " + queueName, metrics); + } + + RES_METRICS.put(queueName, metrics); + } + + return metrics; + } + + private MutableQuantiles planAddReservationLatency; + + private MutableQuantiles planUpdateReservationLatency; + + private MutableQuantiles planDeleteReservationLatency; + + private MutableQuantiles planFollowerSynchronizeLatency; + + private MutableQuantiles reservationAgentCreateReservationLatency; + + private MutableQuantiles reservationAgentUpdateReservationLatency; + + private MutableQuantiles reservationAgentDeleteReservationLatency; + + @Metric("Plan Add Reservation Total Count") + private MutableCounterInt planAddReservationTotalCount; + + @Metric("Plan Update Reservation Total Count") + private MutableCounterInt planUpdateReservationTotalCount; + + @Metric("Plan Delete Reservation Total Count") + private MutableCounterInt planDeleteReservationTotalCount; + + @Metric("Plan Add Reservation Failure Count") + private MutableCounterInt planAddReservationFailureCount; + + @Metric("Plan Update Reservation Failure Count") + private MutableCounterInt planUpdateReservationFailureCount; + + @Metric("Plan Delete Reservation Failure Count") + private MutableCounterInt planDeleteReservationFailureCount; + + @Metric("Plan Follower Synchronize Count") + private MutableCounterInt planFollowerSynchronizeCount; + + private Queue parent; + + private static final MetricsInfo RECORD_INFO = + info("ReservationQueueMetrics", "Reservation Metrics by Queue"); + + private ReservationQueueMetrics(String queueName, Queue parent) { + this.parent = parent; + + registry = new MetricsRegistry(RECORD_INFO); + registry.tag(RECORD_INFO, "ReservationMetricsFor" + queueName); + } + + private void initialize() { + planAddReservationLatency = + registry.newQuantiles("PlanAddReservationLatency", + "Latency for create reservation", "ops", "latency", 60); + + planUpdateReservationLatency = + registry.newQuantiles("PlanUpdateReservationLatency", + "Latency for update reservation", "ops", "latency", 60); + + planDeleteReservationLatency = + registry.newQuantiles("PlanDeleteReservationLatency", + "Latency for remove reservation", "ops", "latency", 60); + + planFollowerSynchronizeLatency = + registry.newQuantiles("PlanFollowerSynchronizeLatency", + "Latency for plan follower execution", "ops", "latency", 5); + + reservationAgentCreateReservationLatency = + registry.newQuantiles("ReservationAgentCreateReservationLatency", + "Latency for create reservation", "ops", "latency", 60); + + reservationAgentUpdateReservationLatency = + registry.newQuantiles("ReservationAgentUpdateReservationLatency", + "Latency for update reservation", "ops", "latency", 60); + + reservationAgentDeleteReservationLatency = + registry.newQuantiles("ReservationAgentDeleteReservationLatency", + "Latency for delete reservation", "ops", "latency", 60); + } + + public void setPlanAddReservationMetrics(long latency, boolean success) { + planAddReservationLatency.add(latency); + planAddReservationTotalCount.incr(); + if (!success) { + planAddReservationFailureCount.incr(); + } + if (parent != null) { + parent.getReservationMetrics().setPlanAddReservationMetrics(latency, + success); + } + } + + public void setPlanUpdateReservationMetrics(long latency, boolean success) { + planUpdateReservationLatency.add(latency); + planUpdateReservationTotalCount.incr(); + if (!success) { + planUpdateReservationFailureCount.incr(); + } + if (parent != null) { + parent.getReservationMetrics().setPlanUpdateReservationMetrics(latency, + success); + } + } + + public void setPlanDeleteReservationMetrics(long latency, boolean success) { + planDeleteReservationLatency.add(latency); + planDeleteReservationTotalCount.incr(); + if (!success) { + planDeleteReservationFailureCount.incr(); + } + if (parent != null) { + parent.getReservationMetrics().setPlanDeleteReservationMetrics(latency, + success); + } + } + + public void setPlanFollowerSynchronizeMetrics(long latency) { + planFollowerSynchronizeLatency.add(latency); + planFollowerSynchronizeCount.incr(); + if (parent != null) { + parent.getReservationMetrics().setPlanFollowerSynchronizeMetrics(latency); + } + } + + public void setAgentCreateReservationMetrics(long latency) { + reservationAgentCreateReservationLatency.add(latency); + if (parent != null) { + parent.getReservationMetrics().setAgentCreateReservationMetrics(latency); + } + } + + public void setAgentUpdateReservationMetrics(long latency) { + reservationAgentUpdateReservationLatency.add(latency); + if (parent != null) { + parent.getReservationMetrics().setAgentUpdateReservationMetrics(latency); + } + } + + public void setAgentDeleteReservationMetrics(long latency) { + reservationAgentDeleteReservationLatency.add(latency); + if (parent != null) { + parent.getReservationMetrics().setAgentDeleteReservationMetrics(latency); + } + } + + protected static StringBuilder sourceName(String queueName) { + StringBuilder sb = new StringBuilder(RECORD_INFO.name()); + int i = 0; + for (String node : Q_SPLITTER.split(queueName)) { + sb.append(",q").append(i++).append('=').append(node); + } + return sb; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Queue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Queue.java index d166e5f..600ce24 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Queue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Queue.java @@ -29,6 +29,7 @@ import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationQueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; @Evolving @@ -47,6 +48,12 @@ QueueMetrics getMetrics(); /** + * Get the queue reservation metrics. + * @return the queue reservation metrics. + */ + ReservationQueueMetrics getReservationMetrics(); + + /** * Get queue information * @param includeChildQueues include child queues? * @param recursive recursively get child queue information? diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java index 43d55c4..5d2ff99 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java @@ -44,6 +44,7 @@ import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.SchedulingRequest; import org.apache.hadoop.yarn.event.EventHandler; +import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationQueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntitlement; @@ -184,6 +185,14 @@ ApplicationResourceUsageReport getAppResourceUsageReport( QueueMetrics getRootQueueMetrics(); /** + * Get the root queue {@link ReservationQueueMetrics} for the scheduler. + * @return the root queue {@link ReservationQueueMetrics} for the scheduler. + */ + @LimitedPrivate("yarn") + @Evolving + ReservationQueueMetrics getRootQueueReservationMetrics(); + + /** * Check if the user has permission to perform the operation. * If the user has {@link QueueACL#ADMINISTER_QUEUE} permission, * this user can view/modify the applications in this queue diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index 9afbdd5..7ed4fd1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -54,6 +54,7 @@ import org.apache.hadoop.yarn.security.PrivilegedEntity.EntityType; import org.apache.hadoop.yarn.security.YarnAuthorizationProvider; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; +import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationQueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.AbsoluteResourceType; @@ -85,6 +86,7 @@ volatile Resource maximumAllocation; private volatile QueueState state = null; final CSQueueMetrics metrics; + private final ReservationQueueMetrics reservationQueueMetrics; protected final PrivilegedEntity queueEntity; final ResourceCalculator resourceCalculator; @@ -149,6 +151,10 @@ public AbstractCSQueue(CapacitySchedulerContext cs, CSQueueMetrics.forQueue(getQueuePath(), parent, cs.getConfiguration().getEnableUserMetrics(), cs.getConf()); + this.reservationQueueMetrics = old != null + ? (ReservationQueueMetrics) old.getReservationMetrics() + : ReservationQueueMetrics.forReservationQueue(getQueuePath(), parent); + this.csContext = cs; this.minimumAllocation = csContext.getMinimumResourceCapability(); @@ -233,7 +239,12 @@ public QueueState getState() { public CSQueueMetrics getMetrics() { return metrics; } - + + @Override + public ReservationQueueMetrics getReservationMetrics() { + return reservationQueueMetrics; + } + @Override public String getQueueName() { return queueName; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index cb01351..787841b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -77,6 +77,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationConstants; +import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationQueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType; @@ -271,6 +272,11 @@ public QueueMetrics getRootQueueMetrics() { return getRootQueue().getMetrics(); } + @Override + public ReservationQueueMetrics getRootQueueReservationMetrics() { + return getRootQueue().getReservationMetrics(); + } + public CSQueue getRootQueue() { return queueManager.getRootQueue(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java index 4babfd5..79a8002 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java @@ -42,6 +42,7 @@ import org.apache.hadoop.yarn.security.PrivilegedEntity; import org.apache.hadoop.yarn.security.PrivilegedEntity.EntityType; import org.apache.hadoop.yarn.security.YarnAuthorizationProvider; +import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationQueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils; import org.apache.hadoop.yarn.util.resource.Resources; @@ -63,6 +64,7 @@ private final YarnAuthorizationProvider authorizer; private final PrivilegedEntity queueEntity; private final FSQueueMetrics metrics; + private final ReservationQueueMetrics reservationQueueMetrics; protected final FSParentQueue parent; protected final RecordFactory recordFactory = @@ -91,6 +93,8 @@ public FSQueue(String name, FairScheduler scheduler, FSParentQueue parent) { YarnAuthorizationProvider.getInstance(scheduler.getConf()); this.queueEntity = new PrivilegedEntity(EntityType.QUEUE, name); this.metrics = FSQueueMetrics.forQueue(getName(), parent, true, scheduler.getConf()); + this.reservationQueueMetrics = + ReservationQueueMetrics.forReservationQueue(getName(), parent); this.parent = parent; setPolicy(scheduler.getAllocationConfiguration().getSchedulingPolicy(name)); reinit(false); @@ -280,6 +284,11 @@ public FSQueueMetrics getMetrics() { return metrics; } + @Override + public ReservationQueueMetrics getReservationMetrics() { + return reservationQueueMetrics; + } + /** Get the fair share assigned to this Schedulable. */ public Resource getFairShare() { return fairShare; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index 1f85814..3962785 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -55,6 +55,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMCriticalThreadUncaughtExceptionHandler; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationConstants; +import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationQueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType; @@ -162,6 +163,7 @@ // Aggregate metrics FSQueueMetrics rootMetrics; + private ReservationQueueMetrics rootReservationMetrics; FSOpDurations fsOpDurations; private float reservableNodesRatio; // percentage of available nodes @@ -1124,6 +1126,11 @@ public QueueMetrics getRootQueueMetrics() { } @Override + public ReservationQueueMetrics getRootQueueReservationMetrics() { + return rootReservationMetrics; + } + + @Override public void handle(SchedulerEvent event) { switch (event.getType()) { case NODE_ADDED: @@ -1331,6 +1338,8 @@ private void initScheduler(Configuration conf) throws IOException { } rootMetrics = FSQueueMetrics.forQueue("root", null, true, conf); + rootReservationMetrics = + ReservationQueueMetrics.forReservationQueue("root", null); fsOpDurations = FSOpDurations.getInstance(true); // This stores per-application scheduling information diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java index 7ac9027..97edce6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java @@ -49,6 +49,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState; +import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationQueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; @@ -121,7 +122,8 @@ private static final String DEFAULT_QUEUE_NAME = "default"; private QueueMetrics metrics; - + private ReservationQueueMetrics reservationMetrics; + private final ResourceCalculator resourceCalculator = new DefaultResourceCalculator(); private final Queue DEFAULT_QUEUE = new Queue() { @@ -136,6 +138,11 @@ public QueueMetrics getMetrics() { } @Override + public ReservationQueueMetrics getReservationMetrics() { + return reservationMetrics; + } + + @Override public QueueInfo getQueueInfo( boolean includeChildQueues, boolean recursive) { QueueInfo queueInfo = recordFactory.newRecordInstance(QueueInfo.class); @@ -249,6 +256,8 @@ private synchronized void initScheduler(Configuration conf) { YarnConfiguration.DEFAULT_RM_SCHEDULER_USE_PORT_FOR_NODE_NAME); this.metrics = QueueMetrics.forQueue(DEFAULT_QUEUE_NAME, null, false, conf); + this.reservationMetrics = + ReservationQueueMetrics.forReservationQueue(DEFAULT_QUEUE_NAME, null); this.activeUsersManager = new ActiveUsersManager(metrics); } @@ -932,6 +941,11 @@ public QueueMetrics getRootQueueMetrics() { } @Override + public ReservationQueueMetrics getRootQueueReservationMetrics() { + return DEFAULT_QUEUE.getReservationMetrics(); + } + + @Override public synchronized boolean checkAccess(UserGroupInformation callerUGI, QueueACL acl, String queueName) { return DEFAULT_QUEUE.hasAccess(acl, callerUGI); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/BaseSharingPolicyTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/BaseSharingPolicyTest.java index b9ce54e..531f6ed 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/BaseSharingPolicyTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/BaseSharingPolicyTest.java @@ -96,7 +96,9 @@ public void setup() { RMContext context = ReservationSystemTestUtil.createMockRMContext(); - plan = new InMemoryPlan(rootQueueMetrics, policy, mAgent, clusterResource, + ReservationQueueMetrics reservationQueueMetrics = mock(ReservationQueueMetrics.class); + + plan = new InMemoryPlan(rootQueueMetrics, reservationQueueMetrics, policy, mAgent, clusterResource, step, res, minAlloc, maxAlloc, "dedicated", null, true, context); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestInMemoryPlan.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestInMemoryPlan.java index c687eea..021462a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestInMemoryPlan.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestInMemoryPlan.java @@ -66,6 +66,7 @@ private Clock clock; private QueueMetrics queueMetrics; + private ReservationQueueMetrics reservationQueueMetrics; private SharingPolicy policy; private ReservationAgent agent; private Planner replanner; @@ -81,6 +82,7 @@ public void setUp() throws PlanningException { clock = mock(Clock.class); queueMetrics = mock(QueueMetrics.class); + reservationQueueMetrics=mock(ReservationQueueMetrics.class); policy = new NoOverCommitPolicy(); replanner = mock(Planner.class); @@ -104,8 +106,9 @@ public void tearDown() { @Test public void testAddReservation() { - Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L, - resCalc, minAlloc, maxAlloc, planName, replanner, true, context); + Plan plan = new InMemoryPlan(queueMetrics,reservationQueueMetrics, policy, agent, + totalCapacity, 1L, resCalc, minAlloc, maxAlloc, planName, replanner, + true, context); ReservationId reservationID = ReservationSystemTestUtil.getNewReservationId(); int[] alloc = { 10, 10, 10, 10, 10, 10 }; @@ -125,8 +128,9 @@ public void testAddReservation() { @Test(expected = PlanningException.class) public void testOutOfRange() throws PlanningException { maxPeriodicity = 100; - Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L, - resCalc, minAlloc, maxAlloc, planName, replanner, true, maxPeriodicity, + Plan plan = new InMemoryPlan(queueMetrics, reservationQueueMetrics, policy, agent, + totalCapacity, 1L, resCalc, minAlloc, maxAlloc, planName, replanner, + true, maxPeriodicity, context, new UTCClock()); // we expect the plan to complaint as the range 330-150 > 50 @@ -138,8 +142,9 @@ public void testOutOfRange() throws PlanningException { public void testAddPeriodicReservation() throws PlanningException { maxPeriodicity = 100; - Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L, - resCalc, minAlloc, maxAlloc, planName, replanner, true, maxPeriodicity, + Plan plan = new InMemoryPlan(queueMetrics, reservationQueueMetrics, policy, agent, + totalCapacity, 1L, resCalc, minAlloc, maxAlloc, planName, replanner, + true, maxPeriodicity, context, new UTCClock()); ReservationId reservationID = @@ -204,8 +209,9 @@ private void checkAllocation(Plan plan, int[] alloc, int start, @Test public void testAddEmptyReservation() { - Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L, - resCalc, minAlloc, maxAlloc, planName, replanner, true, context); + Plan plan = new InMemoryPlan(queueMetrics, reservationQueueMetrics, policy, agent, + totalCapacity, 1L, resCalc, minAlloc, maxAlloc, planName, replanner, + true, context); ReservationId reservationID = ReservationSystemTestUtil.getNewReservationId(); int[] alloc = {}; @@ -223,8 +229,9 @@ public void testAddEmptyReservation() { @Test public void testAddReservationAlreadyExists() { // First add a reservation - Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L, - resCalc, minAlloc, maxAlloc, planName, replanner, true, context); + Plan plan = new InMemoryPlan(queueMetrics, reservationQueueMetrics, policy, agent, + totalCapacity, 1L, resCalc, minAlloc, maxAlloc, planName, replanner, + true, context); ReservationId reservationID = ReservationSystemTestUtil.getNewReservationId(); int[] alloc = { 10, 10, 10, 10, 10, 10 }; @@ -254,8 +261,9 @@ public void testAddReservationAlreadyExists() { @Test public void testUpdateReservation() { - Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L, - resCalc, minAlloc, maxAlloc, planName, replanner, true, context); + Plan plan = new InMemoryPlan(queueMetrics, reservationQueueMetrics, policy, agent, + totalCapacity, 1L, resCalc, minAlloc, maxAlloc, planName, replanner, + true, context); ReservationId reservationID = ReservationSystemTestUtil.getNewReservationId(); // First add a reservation @@ -305,8 +313,10 @@ public void testUpdateReservation() { @Test public void testUpdatePeriodicReservation() { - Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L, - resCalc, minAlloc, maxAlloc, planName, replanner, true, context); + Plan plan = + new InMemoryPlan(queueMetrics, reservationQueueMetrics, policy, agent, + totalCapacity, 1L, resCalc, minAlloc, maxAlloc, planName, replanner, + true, context); // First add a reservation ReservationId reservationID = ReservationSystemTestUtil.getNewReservationId(); @@ -348,8 +358,10 @@ public void testUpdatePeriodicReservation() { @Test public void testUpdateNonExistingReservation() { - Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L, - resCalc, minAlloc, maxAlloc, planName, replanner, true, context); + Plan plan = + new InMemoryPlan(queueMetrics, reservationQueueMetrics, policy, agent, + totalCapacity, 1L, resCalc, minAlloc, maxAlloc, planName, replanner, + true, context); ReservationId reservationID = ReservationSystemTestUtil.getNewReservationId(); // Try to update a reservation without adding @@ -372,8 +384,10 @@ public void testUpdateNonExistingReservation() { @Test public void testDeleteReservation() { // First add a reservation - Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L, - resCalc, minAlloc, maxAlloc, planName, replanner, true, context); + Plan plan = + new InMemoryPlan(queueMetrics, reservationQueueMetrics, policy, agent, + totalCapacity, 1L, resCalc, minAlloc, maxAlloc, planName, replanner, + true, context); ReservationId reservationID = ReservationSystemTestUtil.getNewReservationId(); int[] alloc = { 10, 10, 10, 10, 10, 10 }; @@ -419,8 +433,10 @@ public void testDeleteReservation() { @Test public void testDeletePeriodicReservation() { - Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L, - resCalc, minAlloc, maxAlloc, planName, replanner, true, context); + Plan plan = + new InMemoryPlan(queueMetrics, reservationQueueMetrics, policy, agent, + totalCapacity, 1L, resCalc, minAlloc, maxAlloc, planName, replanner, + true, context); // First add a reservation ReservationId reservationID = ReservationSystemTestUtil.getNewReservationId(); @@ -456,8 +472,10 @@ public void testDeletePeriodicReservation() { @Test public void testDeleteNonExistingReservation() { - Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L, - resCalc, minAlloc, maxAlloc, planName, replanner, true, context); + Plan plan = + new InMemoryPlan(queueMetrics, reservationQueueMetrics, policy, agent, + totalCapacity, 1L, resCalc, minAlloc, maxAlloc, planName, replanner, + true, context); ReservationId reservationID = ReservationSystemTestUtil.getNewReservationId(); // Try to delete a reservation without adding @@ -477,8 +495,9 @@ public void testDeleteNonExistingReservation() { public void testArchiveCompletedReservations() { SharingPolicy sharingPolicy = mock(SharingPolicy.class); Plan plan = - new InMemoryPlan(queueMetrics, sharingPolicy, agent, totalCapacity, 1L, - resCalc, minAlloc, maxAlloc, planName, replanner, true, context); + new InMemoryPlan(queueMetrics, reservationQueueMetrics, sharingPolicy, agent, + totalCapacity, 1L, resCalc, minAlloc, maxAlloc, planName, replanner, + true, context); ReservationId reservationID1 = ReservationSystemTestUtil.getNewReservationId(); // First add a reservation @@ -560,8 +579,10 @@ public void testArchiveCompletedReservations() { @Test public void testGetReservationsById() { - Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L, - resCalc, minAlloc, maxAlloc, planName, replanner, true, context); + Plan plan = + new InMemoryPlan(queueMetrics, reservationQueueMetrics, policy, agent, + totalCapacity, 1L, resCalc, minAlloc, maxAlloc, planName, replanner, + true, context); ReservationId reservationID = ReservationSystemTestUtil.getNewReservationId(); int[] alloc = { 10, 10, 10, 10, 10, 10 }; @@ -593,8 +614,10 @@ public void testGetReservationsById() { @Test public void testGetReservationsByInvalidId() { - Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L, - resCalc, minAlloc, maxAlloc, planName, replanner, true, context); + Plan plan = + new InMemoryPlan(queueMetrics, reservationQueueMetrics, policy, agent, + totalCapacity, 1L, resCalc, minAlloc, maxAlloc, planName, replanner, + true, context); ReservationId reservationID = ReservationSystemTestUtil.getNewReservationId(); int[] alloc = { 10, 10, 10, 10, 10, 10 }; @@ -618,8 +641,10 @@ public void testGetReservationsByInvalidId() { @Test public void testGetReservationsByTimeInterval() { - Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L, - resCalc, minAlloc, maxAlloc, planName, replanner, true, context); + Plan plan = + new InMemoryPlan(queueMetrics, reservationQueueMetrics, policy, agent, + totalCapacity, 1L, resCalc, minAlloc, maxAlloc, planName, replanner, + true, context); ReservationId reservationID = ReservationSystemTestUtil.getNewReservationId(); int[] alloc = { 10, 10, 10, 10, 10, 10 }; @@ -679,8 +704,10 @@ public void testGetReservationsByTimeInterval() { @Test public void testGetReservationsAtTime() { - Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L, - resCalc, minAlloc, maxAlloc, planName, replanner, true, context); + Plan plan = + new InMemoryPlan(queueMetrics, reservationQueueMetrics, policy, agent, + totalCapacity, 1L, resCalc, minAlloc, maxAlloc, planName, replanner, + true, context); ReservationId reservationID = ReservationSystemTestUtil.getNewReservationId(); int[] alloc = { 10, 10, 10, 10, 10, 10 }; @@ -745,7 +772,10 @@ public void testGetReservationReservationFitWithinSearchInterval() { long reservationEnd = Timestamp.valueOf("2050-12-03 10:47:37").getTime(); // Search interval fits the entire reservation but is smaller than the - // period. + // period. Plan plan = + new InMemoryPlan(queueMetrics, reservationQueueMetrics, policy, agent, + totalCapacity, 1L, resCalc, minAlloc, maxAlloc, planName, replanner, + true, context); long searchStart = Timestamp.valueOf("2050-12-03 10:36:37").getTime(); long searchEnd = Timestamp.valueOf("2050-12-03 10:48:37").getTime(); @@ -886,8 +916,10 @@ public void testEnsureReservationEndNotNegativeWhenShifted() { @Test public void testGetReservationsWithNoInput() { - Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L, - resCalc, minAlloc, maxAlloc, planName, replanner, true, context); + Plan plan = + new InMemoryPlan(queueMetrics, reservationQueueMetrics, policy, agent, + totalCapacity, 1L, resCalc, minAlloc, maxAlloc, planName, replanner, + true, context); ReservationId reservationID = ReservationSystemTestUtil.getNewReservationId(); int[] alloc = { 10, 10, 10, 10, 10, 10 }; @@ -912,8 +944,10 @@ public void testGetReservationsWithNoInput() { @Test public void testGetReservationsWithNoReservation() { - Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L, - resCalc, minAlloc, maxAlloc, planName, replanner, true, context); + Plan plan = + new InMemoryPlan(queueMetrics, reservationQueueMetrics, policy, agent, + totalCapacity, 1L, resCalc, minAlloc, maxAlloc, planName, replanner, + true, context); // Verify that get reservation returns no entries if no queries are made. ReservationInterval interval = new ReservationInterval(0, Long.MAX_VALUE); @@ -926,9 +960,10 @@ private void testPositiveGetRecurringReservationsHelper(long reservationStart, long reservationEnd, long searchStart, long searchEnd, long cycles, long period, int periodMultiplier) { maxPeriodicity = period * periodMultiplier; - Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L, - resCalc, minAlloc, maxAlloc, planName, replanner, true, maxPeriodicity, - context, new UTCClock()); + Plan plan = + new InMemoryPlan(queueMetrics, reservationQueueMetrics, policy, agent, + totalCapacity, 1L, resCalc, minAlloc, maxAlloc, planName, replanner, + true, maxPeriodicity, context, new UTCClock()); ReservationId reservationID = submitReservation(plan, reservationStart, reservationEnd, period); @@ -947,9 +982,10 @@ private void testNegativeGetRecurringReservationsHelper(long reservationStart, long reservationEnd, long searchStart, long searchEnd, long cycles, long period, int periodMultiplier) { maxPeriodicity = period * periodMultiplier; - Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L, - resCalc, minAlloc, maxAlloc, planName, replanner, true, maxPeriodicity, - context, new UTCClock()); + Plan plan = + new InMemoryPlan(queueMetrics, reservationQueueMetrics, policy, agent, + totalCapacity, 1L, resCalc, minAlloc, maxAlloc, planName, replanner, + true, maxPeriodicity, context, new UTCClock()); submitReservation(plan, reservationStart, reservationEnd, period); for (int i = 0; i < cycles; i++) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestSchedulerPlanFollowerBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestSchedulerPlanFollowerBase.java index c6cebd9..2d6e557 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestSchedulerPlanFollowerBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestSchedulerPlanFollowerBase.java @@ -59,7 +59,8 @@ protected void testPlanFollower(boolean isMove) throws PlanningException, InterruptedException, AccessControlException { // Initialize plan based on move flag plan = - new InMemoryPlan(scheduler.getRootQueueMetrics(), policy, mAgent, + new InMemoryPlan(scheduler.getRootQueueMetrics(), + scheduler.getRootQueueReservationMetrics(), policy, mAgent, scheduler.getClusterResource(), 1L, res, scheduler.getMinimumResourceCapability(), maxAlloc, "dedicated", null, isMove, context); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestAlignedPlanner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestAlignedPlanner.java index 9fa8559..907bdbd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestAlignedPlanner.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestAlignedPlanner.java @@ -47,6 +47,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.reservation.RLESparseResourceAllocation; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationAllocation; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationInterval; +import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationQueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystemTestUtil; import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException; @@ -1088,6 +1089,8 @@ public void setup() throws Exception { policy.init(reservationQ, conf); QueueMetrics queueMetrics = mock(QueueMetrics.class); + ReservationQueueMetrics reservationQueueMetrics = + mock(ReservationQueueMetrics.class); RMContext context = ReservationSystemTestUtil.createMockRMContext(); conf.setInt(AlignedPlannerWithGreedy.SMOOTHNESS_FACTOR, @@ -1103,8 +1106,9 @@ public void setup() throws Exception { agentLeft.init(conf); // Create Plan - plan = new InMemoryPlan(queueMetrics, policy, agentRight, clusterCapacity, - step, res, minAlloc, maxAlloc, "dedicated", null, true, context); + plan = new InMemoryPlan(queueMetrics, reservationQueueMetrics, policy, + agentRight, clusterCapacity, step, res, minAlloc, maxAlloc, "dedicated", + null, true, context); } private int initializeScenario1() throws PlanningException { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestGreedyReservationAgent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestGreedyReservationAgent.java index 7c7b0a5..34a0806 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestGreedyReservationAgent.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestGreedyReservationAgent.java @@ -47,6 +47,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.reservation.InMemoryReservationAllocation; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationAllocation; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationInterval; +import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationQueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystemTestUtil; import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException; @@ -127,10 +128,13 @@ public void setup() throws Exception { agent.init(conf); QueueMetrics queueMetrics = mock(QueueMetrics.class); + ReservationQueueMetrics reservationQueueMetrics = + mock(ReservationQueueMetrics.class); RMContext context = ReservationSystemTestUtil.createMockRMContext(); - plan = new InMemoryPlan(queueMetrics, policy, agent, clusterCapacity, step, - res, minAlloc, maxAlloc, "dedicated", null, true, context); + plan = new InMemoryPlan(queueMetrics, reservationQueueMetrics, policy, + agent, clusterCapacity, step, res, minAlloc, maxAlloc, "dedicated", + null, true, context); } @SuppressWarnings("javadoc") @@ -750,9 +754,10 @@ public void testStress(int numJobs) throws PlanningException, IOException { policy.init(reservationQ, conf); RMContext context = ReservationSystemTestUtil.createMockRMContext(); - plan = new InMemoryPlan(scheduler.getRootQueueMetrics(), policy, agent, - clusterCapacity, step, res, minAlloc, maxAlloc, "dedicated", null, - true, context); + plan = new InMemoryPlan(scheduler.getRootQueueMetrics(), + scheduler.getRootQueueReservationMetrics(), policy, agent, + clusterCapacity, step, res, minAlloc, maxAlloc, "dedicated", null, true, + context); int acc = 0; List list = new ArrayList(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestReservationAgents.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestReservationAgents.java index 386fa68..561ebef 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestReservationAgents.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestReservationAgents.java @@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.reservation.RLESparseResourceAllocation; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystemTestUtil; +import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationQueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; @@ -136,9 +137,11 @@ public void setup() throws Exception { QueueMetrics queueMetrics = mock(QueueMetrics.class); RMContext context = ReservationSystemTestUtil.createMockRMContext(); + ReservationQueueMetrics reservationQueueMetrics = mock(ReservationQueueMetrics.class); - plan = new InMemoryPlan(queueMetrics, policy, agent, clusterCapacity, step, - resCalc, minAlloc, maxAlloc, "dedicated", null, true, context); + plan = new InMemoryPlan(queueMetrics, reservationQueueMetrics, policy, agent, + clusterCapacity, step, resCalc, minAlloc, maxAlloc, + "dedicated", null, true, context); } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestSimpleCapacityReplanner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestSimpleCapacityReplanner.java index ddd290d..6787c78 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestSimpleCapacityReplanner.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestSimpleCapacityReplanner.java @@ -38,6 +38,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.reservation.NoOverCommitPolicy; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationAllocation; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationInterval; +import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationQueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystemTestUtil; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystemUtil; @@ -67,6 +68,8 @@ public void testReplanningPlanCapacityLoss() throws PlanningException { policy.init("root.dedicated", null); QueueMetrics queueMetrics = mock(QueueMetrics.class); + ReservationQueueMetrics reservationQueueMetrics = + mock(ReservationQueueMetrics.class); when(clock.getTime()).thenReturn(0L); SimpleCapacityReplanner enf = new SimpleCapacityReplanner(clock); @@ -79,7 +82,7 @@ public void testReplanningPlanCapacityLoss() throws PlanningException { enf.init("blah", conf); // Initialize the plan with more resources - InMemoryPlan plan = new InMemoryPlan(queueMetrics, policy, agent, + InMemoryPlan plan = new InMemoryPlan(queueMetrics, reservationQueueMetrics, policy, agent, clusterCapacity, step, res, minAlloc, maxAlloc, "dedicated", enf, true, YarnConfiguration.DEFAULT_RM_RESERVATION_SYSTEM_MAX_PERIODICITY, context, clock);