diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java index 9c04187..63da16c 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java @@ -1322,8 +1322,9 @@ public MyFifoScheduler(RMContext rmContext) { super(); try { Configuration conf = new Configuration(); - reinitialize(conf, rmContext); - } catch (IOException ie) { + setRMContext(rmContext); + reinitialize(conf, null); + } catch (Exception ie) { LOG.info("add application failed with ", ie); assert (false); } diff --git hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java index f923733..90e7e8f 100644 --- hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java +++ hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java @@ -59,6 +59,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; @@ -87,11 +88,12 @@ import com.codahale.metrics.SlidingWindowReservoir; import com.codahale.metrics.Timer; -public class ResourceSchedulerWrapper implements - SchedulerWrapper,ResourceScheduler,Configurable { +public class ResourceSchedulerWrapper extends AbstractYarnScheduler + implements SchedulerWrapper, ResourceScheduler, Configurable { private static final String EOL = System.getProperty("line.separator"); private static final int SAMPLING_SIZE = 60; private ScheduledExecutorService pool; + private RMContext rmContext; // counters for scheduler allocate/handle operations private Counter schedulerAllocateCounter; private Counter schedulerHandleCounter; @@ -144,6 +146,7 @@ public final Logger LOG = Logger.getLogger(ResourceSchedulerWrapper.class); public ResourceSchedulerWrapper() { + super(ResourceSchedulerWrapper.class.getName()); samplerLock = new ReentrantLock(); queueLock = new ReentrantLock(); } @@ -793,9 +796,29 @@ public Configuration getConf() { } @Override - public void reinitialize(Configuration entries, RMContext rmContext) - throws IOException { - scheduler.reinitialize(entries, rmContext); + public void init(Configuration conf) { + ((AbstractYarnScheduler) scheduler).init(conf); + } + + @Override + public void start() { + ((AbstractYarnScheduler) scheduler).start(); + } + + @Override + public void stop() { + ((AbstractYarnScheduler) scheduler).stop(); + } + + @Override + public void setRMContext(RMContext rmContext) { + scheduler.setRMContext(rmContext); + } + + @Override + public void reinitialize(Configuration conf, RMContext rmContext) throws + IOException { + scheduler.reinitialize(conf, null); } @Override diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java index 1d2f376..fe7a968 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java @@ -340,7 +340,7 @@ public RefreshQueuesResponse refreshQueues(RefreshQueuesRequest request) RefreshQueuesResponse response = recordFactory.newRecordInstance(RefreshQueuesResponse.class); try { - rmContext.getScheduler().reinitialize(getConfig(), this.rmContext); + rmContext.getScheduler().reinitialize(getConfig(), null); RMAuditLogger.logSuccess(user.getShortUserName(), argName, "AdminService"); return response; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index b914b1f..b62bd5f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -401,6 +401,8 @@ protected void serviceInit(Configuration configuration) throws Exception { // Initialize the scheduler scheduler = createScheduler(); + scheduler.setRMContext(rmContext); + addIfService(scheduler); rmContext.setScheduler(scheduler); schedulerDispatcher = createSchedulerEventDispatcher(); @@ -429,12 +431,6 @@ protected void serviceInit(Configuration configuration) throws Exception { DefaultMetricsSystem.initialize("ResourceManager"); JvmMetrics.initSingleton("ResourceManager", null); - try { - scheduler.reinitialize(conf, rmContext); - } catch (IOException ioe) { - throw new RuntimeException("Failed to initialize scheduler", ioe); - } - // creating monitors that handle preemption createPolicyMonitors(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java index 0f3af41..3124c17 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java @@ -23,6 +23,7 @@ import java.util.List; import java.util.Map; +import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Container; @@ -33,7 +34,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.util.resource.Resources; -public abstract class AbstractYarnScheduler implements ResourceScheduler { +public abstract class AbstractYarnScheduler extends AbstractService + implements ResourceScheduler { protected RMContext rmContext; protected Map applications; @@ -42,6 +44,15 @@ protected static final Allocation EMPTY_ALLOCATION = new Allocation( EMPTY_CONTAINER_LIST, Resources.createResource(0), null, null, null); + /** + * Construct the service. + * + * @param name service name + */ + public AbstractYarnScheduler(String name) { + super(name); + } + public synchronized List getTransferredContainers( ApplicationAttemptId currentAttempt) { ApplicationId appId = currentAttempt.getApplicationId(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java index 8840881..481bc95 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java @@ -34,9 +34,21 @@ @LimitedPrivate("yarn") @Evolving public interface ResourceScheduler extends YarnScheduler, Recoverable { + + /** + * Set RMContext for ResourceScheduler. + * This method should be called immediately after instantiating + * a scheduler once. + * @param rmContext created by ResourceManager + */ + void setRMContext(RMContext rmContext); + /** * Re-initialize the ResourceScheduler. * @param conf configuration + * @param rmContext this parameter is ignored. Please use + * {@link ResourceScheduler#setRMContext} to set RMContext for + * ResourceScheduler. * @throws IOException */ void reinitialize(Configuration conf, RMContext rmContext) throws IOException; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index e28c18c..d973eac 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -209,7 +209,9 @@ public Configuration getConf() { + ".scheduling-interval-ms"; private static final long DEFAULT_ASYNC_SCHEDULER_INTERVAL = 5; - public CapacityScheduler() {} + public CapacityScheduler() { + super(CapacityScheduler.class.getName()); + } @Override public QueueMetrics getRootQueueMetrics() { @@ -226,7 +228,8 @@ public CapacitySchedulerConfiguration getConfiguration() { } @Override - public RMContainerTokenSecretManager getContainerTokenSecretManager() { + public synchronized RMContainerTokenSecretManager + getContainerTokenSecretManager() { return this.rmContext.getContainerTokenSecretManager(); } @@ -261,7 +264,7 @@ public synchronized int getNumClusterNodes() { } @Override - public RMContext getRMContext() { + public synchronized RMContext getRMContext() { return this.rmContext; } @@ -269,13 +272,34 @@ public RMContext getRMContext() { public Resource getClusterResources() { return clusterResource; } - + + @Override + public synchronized void setRMContext(RMContext rmContext) { + this.rmContext = rmContext; + } + + @Override + public void serviceInit(Configuration conf) throws Exception { + reinitialize(conf, null); + super.serviceInit(conf); + } + + @Override + public void serviceStart() throws Exception { + reinitialize(conf, null); + super.serviceStart(); + } + + @Override + public void serviceStop() throws Exception { + super.serviceStop(); + } + @Override public synchronized void reinitialize(Configuration conf, RMContext rmContext) throws IOException { Configuration configuration = new Configuration(conf); if (!initialized) { - this.rmContext = rmContext; this.conf = loadCapacitySchedulerConfiguration(configuration); validateConf(this.conf); this.minimumAllocation = this.conf.getMinimumAllocation(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java index bedbb64..d1bc9e7 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java @@ -68,7 +68,9 @@ * (this is done to prevent loading a file that hasn't been fully written). */ public static final long ALLOC_RELOAD_WAIT_MS = 5 * 1000; - + + public static final long THREAD_JOIN_TIMEOUT_MS = 1000; + private final Clock clock; private long lastSuccessfulReload; // Last time we successfully reloaded queues @@ -146,7 +148,14 @@ public void run() { @Override public void stop() { running = false; - reloadThread.interrupt(); + if (reloadThread != null) { + reloadThread.interrupt(); + try { + reloadThread.join(THREAD_JOIN_TIMEOUT_MS); + } catch (InterruptedException e) { + LOG.warn("reload Thread fails to join."); + } + } super.stop(); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index fab9ebe..cb3b5d4 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -144,6 +144,11 @@ // How often fair shares are re-calculated (ms) protected long UPDATE_INTERVAL = 500; + private Thread updateThread; + private Thread schedulingThread; + // timeout to join when we stop this service + protected final long THREAD_JOIN_TIMEOUT_MS = 1000; + // Aggregate metrics FSQueueMetrics rootMetrics; @@ -194,6 +199,7 @@ AllocationConfiguration allocConf; public FairScheduler() { + super(FairScheduler.class.getName()); clock = new SystemClock(); allocsLoader = new AllocationFileLoaderService(); queueMgr = new QueueManager(this); @@ -519,7 +525,8 @@ protected Resource resToPreempt(FSLeafQueue sched, long curTime) { return resToPreempt; } - public RMContainerTokenSecretManager getContainerTokenSecretManager() { + public synchronized RMContainerTokenSecretManager + getContainerTokenSecretManager() { return rmContext.getContainerTokenSecretManager(); } @@ -1231,6 +1238,44 @@ public void recover(RMState state) throws Exception { } @Override + public synchronized void setRMContext(RMContext rmContext) { + this.rmContext = rmContext; + } + + @Override + public void serviceInit(Configuration conf) throws Exception { + reinitialize(conf, null); + super.serviceInit(conf); + } + + @Override + public void serviceStart() throws Exception { + reinitialize(conf, null); + super.serviceStart(); + } + + @Override + public void serviceStop() throws Exception { + synchronized (this) { + if (updateThread != null) { + updateThread.interrupt(); + updateThread.join(THREAD_JOIN_TIMEOUT_MS); + } + if (continuousSchedulingEnabled) { + if (schedulingThread != null) { + schedulingThread.interrupt(); + schedulingThread.join(THREAD_JOIN_TIMEOUT_MS); + } + } + if (allocsLoader != null) { + allocsLoader.stop(); + } + } + + super.serviceStop(); + } + + @Override public synchronized void reinitialize(Configuration conf, RMContext rmContext) throws IOException { if (!initialized) { @@ -1255,7 +1300,6 @@ public synchronized void reinitialize(Configuration conf, RMContext rmContext) usePortForNodeName = this.conf.getUsePortForNodeName(); rootMetrics = FSQueueMetrics.forQueue("root", null, true, conf); - this.rmContext = rmContext; // This stores per-application scheduling information this.applications = new ConcurrentHashMap(); @@ -1271,14 +1315,14 @@ public synchronized void reinitialize(Configuration conf, RMContext rmContext) throw new IOException("Failed to start FairScheduler", e); } - Thread updateThread = new Thread(new UpdateThread()); + updateThread = new Thread(new UpdateThread()); updateThread.setName("FairSchedulerUpdateThread"); updateThread.setDaemon(true); updateThread.start(); if (continuousSchedulingEnabled) { // start continuous scheduling thread - Thread schedulingThread = new Thread( + schedulingThread = new Thread( new Runnable() { @Override public void run() { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java index 21fcdec..9c59655 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java @@ -93,6 +93,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent; + import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.server.utils.Lock; import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; @@ -187,6 +188,27 @@ public ActiveUsersManager getActiveUsersManager() { } }; + public FifoScheduler() { + super(FifoScheduler.class.getName()); + } + + @Override + public void serviceInit(Configuration conf) throws Exception { + reinitialize(conf, null); + super.serviceInit(conf); + } + + @Override + public void serviceStart() throws Exception { + reinitialize(conf, null); + super.serviceStart(); + } + + @Override + public void serviceStop() throws Exception { + super.serviceStop(); + } + @Override public synchronized void setConf(Configuration conf) { this.conf = conf; @@ -233,13 +255,17 @@ public Resource getMaximumResourceCapability() { } @Override + public synchronized void setRMContext(RMContext rmContext) { + this.rmContext = rmContext; + } + + @Override public synchronized void reinitialize(Configuration conf, RMContext rmContext) throws IOException { setConf(conf); if (!this.initialized) { validateConf(conf); - this.rmContext = rmContext; //Use ConcurrentSkipListMap because applications need to be ordered this.applications = new ConcurrentSkipListMap(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java index fcd5041..59bf928 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java @@ -218,6 +218,7 @@ public void test() throws Exception { public void testNodeUpdateBeforeAppAttemptInit() throws Exception { FifoScheduler scheduler = new FifoScheduler(); MockRM rm = new MockRM(conf); + scheduler.setRMContext(rm.getRMContext()); scheduler.reinitialize(conf, rm.getRMContext()); RMNode node = MockNodes.newNodeInfo(1, diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index d0ba334..8609725 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -129,8 +129,9 @@ public void testConfValidation() throws Exception { Configuration conf = new YarnConfiguration(); conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 2048); conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 1024); + scheduler.setRMContext(mockContext); try { - scheduler.reinitialize(conf, mockContext); + scheduler.reinitialize(conf, null); fail("Exception is expected because the min memory allocation is" + " larger than the max memory allocation."); } catch (YarnRuntimeException e) { @@ -144,7 +145,7 @@ public void testConfValidation() throws Exception { conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, 2); conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, 1); try { - scheduler.reinitialize(conf, mockContext); + scheduler.reinitialize(conf, null); fail("Exception is expected because the min vcores allocation is" + " larger than the max vcores allocation."); } catch (YarnRuntimeException e) { @@ -342,15 +343,16 @@ public void testRefreshQueues() throws Exception { CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); setupQueueConfiguration(conf); cs.setConf(new YarnConfiguration()); - cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, + cs.setRMContext(new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), new ClientToAMTokenSecretManagerInRM(), null)); + cs.reinitialize(conf, null); checkQueueCapacities(cs, A_CAPACITY, B_CAPACITY); conf.setCapacity(A, 80f); conf.setCapacity(B, 20f); - cs.reinitialize(conf, mockContext); + cs.reinitialize(conf, null); checkQueueCapacities(cs, 80f, 20f); } @@ -441,10 +443,11 @@ public void testParseQueue() throws IOException { conf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".a.a1.b1", 100.0f); conf.setUserLimitFactor(CapacitySchedulerConfiguration.ROOT + ".a.a1.b1", 100.0f); - cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, + cs.setRMContext(new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), new ClientToAMTokenSecretManagerInRM(), null)); + cs.reinitialize(conf, null); } @Test @@ -454,10 +457,11 @@ public void testReconnectedNode() throws Exception { setupQueueConfiguration(csConf); CapacityScheduler cs = new CapacityScheduler(); cs.setConf(new YarnConfiguration()); - cs.reinitialize(csConf, new RMContextImpl(null, null, null, null, + cs.setRMContext(new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(csConf), new NMTokenSecretManagerInRM(csConf), new ClientToAMTokenSecretManagerInRM(), null)); + cs.reinitialize(csConf, null); RMNode n1 = MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1); RMNode n2 = MockNodes.newNodeInfo(0, MockNodes.newResource(2 * GB), 2); @@ -481,10 +485,11 @@ public void testRefreshQueuesWithNewQueue() throws Exception { CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); setupQueueConfiguration(conf); cs.setConf(new YarnConfiguration()); - cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, + cs.setRMContext(new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), new ClientToAMTokenSecretManagerInRM(), null)); + cs.reinitialize(conf, null); checkQueueCapacities(cs, A_CAPACITY, B_CAPACITY); // Add a new queue b4 @@ -500,7 +505,7 @@ public void testRefreshQueuesWithNewQueue() throws Exception { conf.setCapacity(B2, B2_CAPACITY); conf.setCapacity(B3, B3_CAPACITY); conf.setCapacity(B4, B4_CAPACITY); - cs.reinitialize(conf,mockContext); + cs.reinitialize(conf, null); checkQueueCapacities(cs, 80f, 20f); // Verify parent for B4 @@ -632,10 +637,11 @@ public void testAddAndRemoveAppFromCapacityScheduler() throws Exception { CapacityScheduler cs = new CapacityScheduler(); CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); setupQueueConfiguration(conf); - cs.reinitialize(conf, new RMContextImpl(rmDispatcher, null, null, null, + cs.setRMContext(new RMContextImpl(rmDispatcher, null, null, null, null, null, new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), new ClientToAMTokenSecretManagerInRM(), null)); + cs.reinitialize(conf, null); SchedulerApplication app = TestSchedulerUtils.verifyAppAddedAndRemovedFromScheduler( diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index bdf89bb..0d0f385 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -142,7 +142,8 @@ public void setUp() throws Exception { queues, queues, TestUtils.spyHook); - cs.reinitialize(csConf, rmContext); + cs.setRMContext(rmContext); + cs.serviceInit(csConf); } private static final String A = "a"; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java index f0e2c04..84c58a6 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java @@ -44,10 +44,11 @@ public void testQueueParsing() throws Exception { CapacityScheduler capacityScheduler = new CapacityScheduler(); capacityScheduler.setConf(conf); - capacityScheduler.reinitialize(conf, new RMContextImpl(null, null, + capacityScheduler.setRMContext(new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), new ClientToAMTokenSecretManagerInRM(), null)); + capacityScheduler.reinitialize(conf, null); CSQueue a = capacityScheduler.getQueue("a"); Assert.assertEquals(0.10, a.getAbsoluteCapacity(), DELTA); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java index e4dc801..1d0d95f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java @@ -40,7 +40,7 @@ private Resource maxResource = Resources.createResource(10); @Before - public void setup() throws IOException { + public void setup() throws Exception { FairScheduler scheduler = new FairScheduler(); Configuration conf = createConfiguration(); // All tests assume only one assignment per node update @@ -48,7 +48,8 @@ public void setup() throws IOException { ResourceManager resourceManager = new ResourceManager(); resourceManager.init(conf); ((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start(); - scheduler.reinitialize(conf, resourceManager.getRMContext()); + scheduler.serviceInit(conf); + scheduler.setRMContext(resourceManager.getRMContext()); String queueName = "root.queue1"; scheduler.allocConf = mock(AllocationConfiguration.class); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 2524763..9cb4faa 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -151,6 +151,9 @@ public void setUp() throws IOException { // to initialize the master key resourceManager.getRMContext().getContainerTokenSecretManager().rollMasterKey(); + + // To initialize scheduler + scheduler.setRMContext(resourceManager.getRMContext()); } @After @@ -316,7 +319,7 @@ public void testLoadConfigurationOnInitialize() throws IOException { conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 512); conf.setInt(FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 128); - scheduler.reinitialize(conf, resourceManager.getRMContext()); + scheduler.reinitialize(conf, null); Assert.assertEquals(true, scheduler.assignMultiple); Assert.assertEquals(3, scheduler.maxAssign); Assert.assertEquals(true, scheduler.sizeBasedWeight); @@ -360,7 +363,7 @@ public void testMinZeroResourcesSettings() throws IOException { FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 512); conf.setInt( FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_VCORES, 2); - fs.reinitialize(conf, null); + fs.reinitialize(conf, null); Assert.assertEquals(0, fs.getMinimumResourceCapability().getMemory()); Assert.assertEquals(0, fs.getMinimumResourceCapability().getVirtualCores()); Assert.assertEquals(512, fs.getIncrementResourceCapability().getMemory()); @@ -369,7 +372,7 @@ public void testMinZeroResourcesSettings() throws IOException { @Test public void testAggregateCapacityTracking() throws Exception { - scheduler.reinitialize(conf, resourceManager.getRMContext()); + scheduler.reinitialize(conf, null); // Add a node RMNode node1 = @@ -394,7 +397,7 @@ public void testAggregateCapacityTracking() throws Exception { @Test public void testSimpleFairShareCalculation() throws IOException { - scheduler.reinitialize(conf, resourceManager.getRMContext()); + scheduler.reinitialize(conf, null); // Add one big node (only care about aggregate capacity) RMNode node1 = @@ -421,7 +424,7 @@ public void testSimpleFairShareCalculation() throws IOException { @Test public void testSimpleHierarchicalFairShareCalculation() throws IOException { - scheduler.reinitialize(conf, resourceManager.getRMContext()); + scheduler.reinitialize(conf, null); // Add one big node (only care about aggregate capacity) int capacity = 10 * 24; @@ -454,7 +457,7 @@ public void testSimpleHierarchicalFairShareCalculation() throws IOException { @Test public void testHierarchicalQueuesSimilarParents() throws IOException { - scheduler.reinitialize(conf, resourceManager.getRMContext()); + scheduler.reinitialize(conf, null); QueueManager queueManager = scheduler.getQueueManager(); FSLeafQueue leafQueue = queueManager.getLeafQueue("parent.child", true); @@ -478,7 +481,7 @@ public void testHierarchicalQueuesSimilarParents() throws IOException { @Test public void testSchedulerRootQueueMetrics() throws Exception { - scheduler.reinitialize(conf, resourceManager.getRMContext()); + scheduler.reinitialize(conf, null); // Add a node RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(1024)); @@ -517,7 +520,7 @@ public void testSchedulerRootQueueMetrics() throws Exception { @Test (timeout = 5000) public void testSimpleContainerAllocation() throws IOException { - scheduler.reinitialize(conf, resourceManager.getRMContext()); + scheduler.reinitialize(conf, null); // Add a node RMNode node1 = @@ -565,7 +568,7 @@ public void testSimpleContainerAllocation() throws IOException { @Test (timeout = 5000) public void testSimpleContainerReservation() throws Exception { - scheduler.reinitialize(conf, resourceManager.getRMContext()); + scheduler.reinitialize(conf, null); // Add a node RMNode node1 = @@ -619,7 +622,7 @@ public void testSimpleContainerReservation() throws Exception { @Test public void testUserAsDefaultQueue() throws Exception { conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "true"); - scheduler.reinitialize(conf, resourceManager.getRMContext()); + scheduler.reinitialize(conf, null); RMContext rmContext = resourceManager.getRMContext(); Map appsMap = rmContext.getRMApps(); ApplicationAttemptId appAttemptId = createAppAttemptId(1, 1); @@ -645,7 +648,7 @@ public void testUserAsDefaultQueue() throws Exception { @Test public void testNotUserAsDefaultQueue() throws Exception { conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "false"); - scheduler.reinitialize(conf, resourceManager.getRMContext()); + scheduler.reinitialize(conf, null); RMContext rmContext = resourceManager.getRMContext(); Map appsMap = rmContext.getRMApps(); ApplicationAttemptId appAttemptId = createAppAttemptId(1, 1); @@ -671,7 +674,7 @@ public void testNotUserAsDefaultQueue() throws Exception { @Test public void testEmptyQueueName() throws Exception { - scheduler.reinitialize(conf, resourceManager.getRMContext()); + scheduler.reinitialize(conf, null); // only default queue assertEquals(1, scheduler.getQueueManager().getLeafQueues().size()); @@ -691,7 +694,7 @@ public void testEmptyQueueName() throws Exception { @Test public void testAssignToQueue() throws Exception { conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "true"); - scheduler.reinitialize(conf, resourceManager.getRMContext()); + scheduler.reinitialize(conf, null); RMApp rmApp1 = new MockRMApp(0, 0, RMAppState.NEW); RMApp rmApp2 = new MockRMApp(1, 1, RMAppState.NEW); @@ -709,7 +712,7 @@ public void testAssignToQueue() throws Exception { @Test public void testAssignToNonLeafQueueReturnsNull() throws Exception { conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "true"); - scheduler.reinitialize(conf, resourceManager.getRMContext()); + scheduler.reinitialize(conf, null); scheduler.getQueueManager().getLeafQueue("root.child1.granchild", true); scheduler.getQueueManager().getLeafQueue("root.child2", true); @@ -726,7 +729,7 @@ public void testAssignToNonLeafQueueReturnsNull() throws Exception { public void testQueuePlacementWithPolicy() throws Exception { conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, SimpleGroupsMapping.class, GroupMappingServiceProvider.class); - scheduler.reinitialize(conf, resourceManager.getRMContext()); + scheduler.reinitialize(conf, null); ApplicationAttemptId appId; @@ -787,7 +790,7 @@ public void testFairShareWithMinAlloc() throws Exception { out.println(""); out.close(); - scheduler.reinitialize(conf, resourceManager.getRMContext()); + scheduler.reinitialize(conf, null); // Add one big node (only care about aggregate capacity) RMNode node1 = @@ -835,7 +838,7 @@ public void testNestedUserQueue() throws IOException { out.println(""); out.close(); - scheduler.reinitialize(conf, resourceManager.getRMContext()); + scheduler.reinitialize(conf, null); RMApp rmApp1 = new MockRMApp(0, 0, RMAppState.NEW); FSLeafQueue user1Leaf = scheduler.assignToQueue(rmApp1, "root.default", @@ -867,7 +870,7 @@ public void testFairShareAndWeightsInNestedUserQueueRule() throws Exception { RMApp rmApp1 = new MockRMApp(0, 0, RMAppState.NEW); RMApp rmApp2 = new MockRMApp(1, 1, RMAppState.NEW); - scheduler.reinitialize(conf, resourceManager.getRMContext()); + scheduler.reinitialize(conf, null); int capacity = 16 * 1024; // create node with 16 G @@ -901,7 +904,7 @@ public void testFairShareAndWeightsInNestedUserQueueRule() throws Exception { */ @Test public void testQueueDemandCalculation() throws Exception { - scheduler.reinitialize(conf, resourceManager.getRMContext()); + scheduler.reinitialize(conf, null); ApplicationAttemptId id11 = createAppAttemptId(1, 1); scheduler.addApplication(id11.getApplicationId(), "root.queue1", "user1"); @@ -951,7 +954,7 @@ public void testQueueDemandCalculation() throws Exception { @Test public void testAppAdditionAndRemoval() throws Exception { - scheduler.reinitialize(conf, resourceManager.getRMContext()); + scheduler.reinitialize(conf, null); ApplicationAttemptId attemptId =createAppAttemptId(1, 1); AppAddedSchedulerEvent appAddedEvent = new AppAddedSchedulerEvent(attemptId.getApplicationId(), "default", "user1"); @@ -1001,7 +1004,7 @@ public void testHierarchicalQueueAllocationFileParsing() throws IOException, SAX out.println(""); out.close(); - scheduler.reinitialize(conf, resourceManager.getRMContext()); + scheduler.reinitialize(conf, null); QueueManager queueManager = scheduler.getQueueManager(); Collection leafQueues = queueManager.getLeafQueues(); @@ -1034,7 +1037,7 @@ public void testConfigureRootQueue() throws Exception { out.println(""); out.close(); - scheduler.reinitialize(conf, resourceManager.getRMContext()); + scheduler.reinitialize(conf, null); QueueManager queueManager = scheduler.getQueueManager(); FSQueue root = queueManager.getRootQueue(); @@ -1060,7 +1063,7 @@ public void testIsStarvedForMinShare() throws Exception { out.println(""); out.close(); - scheduler.reinitialize(conf, resourceManager.getRMContext()); + scheduler.reinitialize(conf, null); // Add one big node (only care about aggregate capacity) RMNode node1 = @@ -1117,7 +1120,7 @@ public void testIsStarvedForFairShare() throws Exception { out.println(""); out.close(); - scheduler.reinitialize(conf, resourceManager.getRMContext()); + scheduler.reinitialize(conf, null); // Add one big node (only care about aggregate capacity) RMNode node1 = @@ -1190,7 +1193,7 @@ public void testChoiceOfPreemptedContainers() throws Exception { out.println(""); out.close(); - scheduler.reinitialize(conf, resourceManager.getRMContext()); + scheduler.reinitialize(conf, null); // Create four nodes RMNode node1 = @@ -1357,7 +1360,7 @@ public void testPreemptionDecision() throws Exception { out.println(""); out.close(); - scheduler.reinitialize(conf, resourceManager.getRMContext()); + scheduler.reinitialize(conf, null); // Create four nodes RMNode node1 = @@ -1454,7 +1457,7 @@ public void testPreemptionDecision() throws Exception { @Test (timeout = 5000) public void testMultipleContainersWaitingForReservation() throws IOException { - scheduler.reinitialize(conf, resourceManager.getRMContext()); + scheduler.reinitialize(conf, null); // Add a node RMNode node1 = @@ -1496,7 +1499,7 @@ public void testUserMaxRunningApps() throws Exception { out.println(""); out.close(); - scheduler.reinitialize(conf, resourceManager.getRMContext()); + scheduler.reinitialize(conf, null); // Add a node RMNode node1 = @@ -1537,7 +1540,7 @@ public void testUserMaxRunningApps() throws Exception { @Test (timeout = 5000) public void testReservationWhileMultiplePriorities() throws IOException { - scheduler.reinitialize(conf, resourceManager.getRMContext()); + scheduler.reinitialize(conf, null); // Add a node RMNode node1 = @@ -1617,7 +1620,7 @@ public void testAclSubmitApplication() throws Exception { out.println(""); out.close(); - scheduler.reinitialize(conf, resourceManager.getRMContext()); + scheduler.reinitialize(conf, null); ApplicationAttemptId attId1 = createSchedulingRequest(1024, "queue1", "norealuserhasthisname", 1); @@ -1632,7 +1635,7 @@ public void testAclSubmitApplication() throws Exception { @Test (timeout = 5000) public void testMultipleNodesSingleRackRequest() throws Exception { - scheduler.reinitialize(conf, resourceManager.getRMContext()); + scheduler.reinitialize(conf, null); RMNode node1 = MockNodes @@ -1681,7 +1684,7 @@ public void testMultipleNodesSingleRackRequest() throws Exception { @Test (timeout = 5000) public void testFifoWithinQueue() throws Exception { - scheduler.reinitialize(conf, resourceManager.getRMContext()); + scheduler.reinitialize(conf, null); RMNode node1 = MockNodes @@ -1725,7 +1728,7 @@ public void testFifoWithinQueue() throws Exception { @Test(timeout = 3000) public void testMaxAssign() throws Exception { conf.setBoolean(FairSchedulerConfiguration.ASSIGN_MULTIPLE, true); - scheduler.reinitialize(conf, resourceManager.getRMContext()); + scheduler.reinitialize(conf, null); RMNode node = MockNodes.newNodeInfo(1, Resources.createResource(16384, 16), 0, @@ -1768,7 +1771,7 @@ public void testMaxAssign() throws Exception { */ @Test(timeout = 5000) public void testAssignContainer() throws Exception { - scheduler.reinitialize(conf, resourceManager.getRMContext()); + scheduler.reinitialize(conf, null); final String user = "user1"; final String fifoQueue = "fifo"; @@ -1852,7 +1855,7 @@ public void testNotAllowSubmitApplication() throws Exception { out.println(""); out.close(); - scheduler.reinitialize(conf, resourceManager.getRMContext()); + scheduler.reinitialize(conf, null); int appId = this.APP_ID++; String user = "usernotallow"; @@ -1902,7 +1905,7 @@ public void testNotAllowSubmitApplication() throws Exception { @Test public void testReservationThatDoesntFit() throws IOException { - scheduler.reinitialize(conf, resourceManager.getRMContext()); + scheduler.reinitialize(conf, null); RMNode node1 = MockNodes @@ -1930,7 +1933,7 @@ public void testReservationThatDoesntFit() throws IOException { @Test public void testRemoveNodeUpdatesRootQueueMetrics() throws IOException { - scheduler.reinitialize(conf, resourceManager.getRMContext()); + scheduler.reinitialize(conf, null); assertEquals(0, scheduler.getRootQueueMetrics().getAvailableMB()); assertEquals(0, scheduler.getRootQueueMetrics().getAvailableVirtualCores()); @@ -1958,7 +1961,7 @@ public void testRemoveNodeUpdatesRootQueueMetrics() throws IOException { @Test public void testStrictLocality() throws IOException { - scheduler.reinitialize(conf, resourceManager.getRMContext()); + scheduler.reinitialize(conf, null); RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(1024), 1, "127.0.0.1"); NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); @@ -1998,7 +2001,7 @@ public void testStrictLocality() throws IOException { @Test public void testCancelStrictLocality() throws IOException { - scheduler.reinitialize(conf, resourceManager.getRMContext()); + scheduler.reinitialize(conf, null); RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(1024), 1, "127.0.0.1"); NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); @@ -2048,7 +2051,7 @@ public void testCancelStrictLocality() throws IOException { */ @Test public void testReservationsStrictLocality() throws IOException { - scheduler.reinitialize(conf, resourceManager.getRMContext()); + scheduler.reinitialize(conf, null); RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(1024), 1, "127.0.0.1"); RMNode node2 = MockNodes.newNodeInfo(1, Resources.createResource(1024), 2, "127.0.0.2"); @@ -2088,7 +2091,7 @@ public void testReservationsStrictLocality() throws IOException { @Test public void testNoMoreCpuOnNode() throws IOException { - scheduler.reinitialize(conf, resourceManager.getRMContext()); + scheduler.reinitialize(conf, null); RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(2048, 1), 1, "127.0.0.1"); @@ -2109,7 +2112,7 @@ public void testNoMoreCpuOnNode() throws IOException { @Test public void testBasicDRFAssignment() throws Exception { - scheduler.reinitialize(conf, resourceManager.getRMContext()); + scheduler.reinitialize(conf, null); RMNode node = MockNodes.newNodeInfo(1, BuilderUtils.newResource(8192, 5)); NodeAddedSchedulerEvent nodeEvent = new NodeAddedSchedulerEvent(node); @@ -2149,7 +2152,7 @@ public void testBasicDRFAssignment() throws Exception { */ @Test public void testBasicDRFWithQueues() throws Exception { - scheduler.reinitialize(conf, resourceManager.getRMContext()); + scheduler.reinitialize(conf, null); RMNode node = MockNodes.newNodeInfo(1, BuilderUtils.newResource(8192, 7), 1, "127.0.0.1"); @@ -2185,7 +2188,7 @@ public void testBasicDRFWithQueues() throws Exception { @Test public void testDRFHierarchicalQueues() throws Exception { - scheduler.reinitialize(conf, resourceManager.getRMContext()); + scheduler.reinitialize(conf, null); RMNode node = MockNodes.newNodeInfo(1, BuilderUtils.newResource(12288, 12), 1, "127.0.0.1"); @@ -2253,8 +2256,7 @@ public void testDRFHierarchicalQueues() throws Exception { public void testHostPortNodeName() throws Exception { conf.setBoolean(YarnConfiguration .RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME, true); - scheduler.reinitialize(conf, - resourceManager.getRMContext()); + scheduler.reinitialize(conf, null); RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(1024), 1, "127.0.0.1", 1); NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); @@ -2334,7 +2336,7 @@ public void testUserAndQueueMaxRunningApps() throws Exception { out.println(""); out.close(); - scheduler.reinitialize(conf, resourceManager.getRMContext()); + scheduler.reinitialize(conf, null); // exceeds no limits ApplicationAttemptId attId1 = createSchedulingRequest(1024, "queue1", "user1"); @@ -2388,7 +2390,7 @@ public void testMaxRunningAppsHierarchicalQueues() throws Exception { out.println(""); out.close(); - scheduler.reinitialize(conf, resourceManager.getRMContext()); + scheduler.reinitialize(conf, null); // exceeds no limits ApplicationAttemptId attId1 = createSchedulingRequest(1024, "queue1.sub1", "user1"); @@ -2449,7 +2451,8 @@ public void testContinuousScheduling() throws Exception { Configuration conf = createConfiguration(); conf.setBoolean(FairSchedulerConfiguration.CONTINUOUS_SCHEDULING_ENABLED, true); - fs.reinitialize(conf, resourceManager.getRMContext()); + fs.setRMContext(resourceManager.getRMContext()); + fs.reinitialize(conf, null); Assert.assertTrue("Continuous scheduling should be enabled.", fs.isContinuousSchedulingEnabled()); @@ -2529,7 +2532,7 @@ public void testDontAllowUndeclaredPools() throws Exception{ out.println(""); out.close(); - scheduler.reinitialize(conf, resourceManager.getRMContext()); + scheduler.reinitialize(conf, null); QueueManager queueManager = scheduler.getQueueManager(); FSLeafQueue jerryQueue = queueManager.getLeafQueue("jerry", false); @@ -2559,7 +2562,7 @@ public void testDontAllowUndeclaredPools() throws Exception{ @SuppressWarnings("resource") @Test public void testBlacklistNodes() throws Exception { - scheduler.reinitialize(conf, resourceManager.getRMContext()); + scheduler.reinitialize(conf, null); final int GB = 1024; String host = "127.0.0.1"; @@ -2611,7 +2614,7 @@ public void testBlacklistNodes() throws Exception { @Test public void testGetAppsInQueue() throws Exception { - scheduler.reinitialize(conf, resourceManager.getRMContext()); + scheduler.reinitialize(conf, null); ApplicationAttemptId appAttId1 = createSchedulingRequest(1024, 1, "queue1.subqueue1", "user1"); @@ -2655,7 +2658,7 @@ public void testAddAndRemoveAppFromFairScheduler() throws Exception { @Test public void testMoveRunnableApp() throws Exception { - scheduler.reinitialize(conf, resourceManager.getRMContext()); + scheduler.reinitialize(conf, null); QueueManager queueMgr = scheduler.getQueueManager(); FSLeafQueue oldQueue = queueMgr.getLeafQueue("queue1", true); @@ -2694,7 +2697,7 @@ public void testMoveRunnableApp() throws Exception { @Test public void testMoveNonRunnableApp() throws Exception { - scheduler.reinitialize(conf, resourceManager.getRMContext()); + scheduler.reinitialize(conf, null); QueueManager queueMgr = scheduler.getQueueManager(); FSLeafQueue oldQueue = queueMgr.getLeafQueue("queue1", true); @@ -2714,7 +2717,7 @@ public void testMoveNonRunnableApp() throws Exception { @Test public void testMoveMakesAppRunnable() throws Exception { - scheduler.reinitialize(conf, resourceManager.getRMContext()); + scheduler.reinitialize(conf, null); QueueManager queueMgr = scheduler.getQueueManager(); FSLeafQueue oldQueue = queueMgr.getLeafQueue("queue1", true); @@ -2741,7 +2744,7 @@ public void testMoveMakesAppRunnable() throws Exception { @Test (expected = YarnException.class) public void testMoveWouldViolateMaxAppsConstraints() throws Exception { - scheduler.reinitialize(conf, resourceManager.getRMContext()); + scheduler.reinitialize(conf, null); QueueManager queueMgr = scheduler.getQueueManager(); queueMgr.getLeafQueue("queue2", true); @@ -2755,7 +2758,7 @@ public void testMoveWouldViolateMaxAppsConstraints() throws Exception { @Test (expected = YarnException.class) public void testMoveWouldViolateMaxResourcesConstraints() throws Exception { - scheduler.reinitialize(conf, resourceManager.getRMContext()); + scheduler.reinitialize(conf, null); QueueManager queueMgr = scheduler.getQueueManager(); FSLeafQueue oldQueue = queueMgr.getLeafQueue("queue1", true); @@ -2778,7 +2781,7 @@ public void testMoveWouldViolateMaxResourcesConstraints() throws Exception { @Test (expected = YarnException.class) public void testMoveToNonexistentQueue() throws Exception { - scheduler.reinitialize(conf, resourceManager.getRMContext()); + scheduler.reinitialize(conf, null); scheduler.getQueueManager().getLeafQueue("queue1", true); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java index 77a3b02..fb11ef6 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java @@ -38,7 +38,7 @@ private ResourceManager resourceManager; @Before - public void setUp() throws IOException { + public void setUp() throws Exception { scheduler = new FairScheduler(); Configuration conf = new YarnConfiguration(); @@ -51,7 +51,8 @@ public void setUp() throws IOException { resourceManager = new ResourceManager(); resourceManager.init(conf); ((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start(); - scheduler.reinitialize(conf, resourceManager.getRMContext()); + scheduler.serviceInit(conf); + scheduler.setRMContext(resourceManager.getRMContext()); } /** diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java index f6dfc3f..f050f5f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java @@ -148,7 +148,8 @@ public void testAppAttemptMetrics() throws Exception { null, null, null, null, null, null, null, writer); FifoScheduler schedular = new FifoScheduler(); - schedular.reinitialize(new Configuration(), rmContext); + schedular.setRMContext(rmContext); + schedular.reinitialize(new Configuration(), null); QueueMetrics metrics = schedular.getRootQueueMetrics(); int beforeAppsSubmitted = metrics.getAppsSubmitted(); @@ -186,7 +187,8 @@ public void testNodeLocalAssignment() throws Exception { null, containerTokenSecretManager, nmTokenSecretManager, null, writer); FifoScheduler scheduler = new FifoScheduler(); - scheduler.reinitialize(new Configuration(), rmContext); + scheduler.setRMContext(rmContext); + scheduler.reinitialize(new Configuration(), null); RMNode node0 = MockNodes.newNodeInfo(1, Resources.createResource(1024 * 64), 1, "127.0.0.1"); @@ -256,7 +258,8 @@ public void testUpdateResourceOnNode() throws Exception { return nodes; } }; - scheduler.reinitialize(new Configuration(), rmContext); + scheduler.setRMContext(rmContext); + scheduler.reinitialize(new Configuration(), null); RMNode node0 = MockNodes.newNodeInfo(1, Resources.createResource(2048, 4), 1, "127.0.0.1"); NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node0); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java index 2c2aae6..f37b48a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java @@ -203,10 +203,11 @@ public static CapacityScheduler mockCapacityScheduler() throws IOException { CapacityScheduler cs = new CapacityScheduler(); cs.setConf(new YarnConfiguration()); - cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, + cs.setRMContext(new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), new ClientToAMTokenSecretManagerInRM(), null)); + cs.reinitialize(conf, null); return cs; }