diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java index 4b5fa0adee3..75798876dda 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java @@ -130,6 +130,7 @@ import org.apache.hadoop.yarn.server.api.records.NodeAction; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore; @@ -189,7 +190,7 @@ public void testSimple() throws Exception { rm.start(); // Submit the application - RMApp app = rm.submitApp(1024); + RMApp app = MockRMAppSubmitter.submitWithMemory(1024, rm); rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); @@ -277,7 +278,7 @@ public void testMapNodeLocality() throws Exception { rm.start(); // Submit the application - RMApp app = rm.submitApp(1024); + RMApp app = MockRMAppSubmitter.submitWithMemory(1024, rm); rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); @@ -358,7 +359,7 @@ public void testResource() throws Exception { rm.start(); // Submit the application - RMApp app = rm.submitApp(1024); + RMApp app = MockRMAppSubmitter.submitWithMemory(1024, rm); rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); @@ -422,7 +423,7 @@ public void testReducerRampdownDiagnostics() throws Exception { conf.setFloat(MRJobConfig.COMPLETED_MAPS_FOR_REDUCE_SLOWSTART, 0.0f); final MyResourceManager rm = new MyResourceManager(conf); rm.start(); - final RMApp app = rm.submitApp(1024); + final RMApp app = MockRMAppSubmitter.submitWithMemory(1024, rm); rm.drainEvents(); final String host = "host1"; @@ -478,7 +479,7 @@ public void testPreemptReducers() throws Exception { rm.start(); // Submit the application - RMApp app = rm.submitApp(1024); + RMApp app = MockRMAppSubmitter.submitWithMemory(1024, rm); rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); @@ -530,7 +531,7 @@ public void testNonAggressivelyPreemptReducers() throws Exception { rm.start(); // Submit the application - RMApp app = rm.submitApp(1024); + RMApp app = MockRMAppSubmitter.submitWithMemory(1024, rm); rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); @@ -594,7 +595,7 @@ public void testUnconditionalPreemptReducers() throws Exception { rm.getMyFifoScheduler().forceResourceLimit(Resource.newInstance(8192, 8)); // Submit the application - RMApp app = rm.submitApp(1024); + RMApp app = MockRMAppSubmitter.submitWithMemory(1024, rm); rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); @@ -648,7 +649,7 @@ public void testExcessReduceContainerAssign() throws Exception { conf.setFloat(MRJobConfig.COMPLETED_MAPS_FOR_REDUCE_SLOWSTART, 0.0f); final MyResourceManager2 rm = new MyResourceManager2(conf); rm.start(); - final RMApp app = rm.submitApp(2048); + final RMApp app = MockRMAppSubmitter.submitWithMemory(2048, rm); rm.drainEvents(); final String host = "host1"; final MockNM nm = rm.registerNode(String.format("%s:1234", host), 4096); @@ -879,7 +880,7 @@ public void testMapReduceScheduling() throws Exception { rm.start(); // Submit the application - RMApp app = rm.submitApp(1024); + RMApp app = MockRMAppSubmitter.submitWithMemory(1024, rm); rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); @@ -1011,7 +1012,7 @@ public void testReportedAppProgress() throws Exception { .getDispatcher(); // Submit the application - RMApp rmApp = rm.submitApp(1024); + RMApp rmApp = MockRMAppSubmitter.submitWithMemory(1024, rm); rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 21504); @@ -1163,7 +1164,7 @@ public void testReportedAppProgressWithOnlyMaps() throws Exception { .getDispatcher(); // Submit the application - RMApp rmApp = rm.submitApp(1024); + RMApp rmApp = MockRMAppSubmitter.submitWithMemory(1024, rm); rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 11264); @@ -1256,7 +1257,7 @@ public void testUpdatedNodes() throws Exception { rm.start(); // Submit the application - RMApp app = rm.submitApp(1024); + RMApp app = MockRMAppSubmitter.submitWithMemory(1024, rm); rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); amNodeManager.nodeHeartbeat(true); @@ -1351,7 +1352,7 @@ public void testBlackListedNodes() throws Exception { rm.start(); // Submit the application - RMApp app = rm.submitApp(1024); + RMApp app = MockRMAppSubmitter.submitWithMemory(1024, rm); rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); @@ -1460,7 +1461,7 @@ public void testIgnoreBlacklisting() throws Exception { rm.start(); // Submit the application - RMApp app = rm.submitApp(1024); + RMApp app = MockRMAppSubmitter.submitWithMemory(1024, rm); rm.drainEvents(); MockNM[] nodeManagers = new MockNM[10]; @@ -1649,7 +1650,7 @@ public void testBlackListedNodesWithSchedulingToThatNode() throws Exception { rm.start(); // Submit the application - RMApp app = rm.submitApp(1024); + RMApp app = MockRMAppSubmitter.submitWithMemory(1024, rm); rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); @@ -2322,7 +2323,7 @@ public void testCompletedTasksRecalculateSchedule() throws Exception { rm.start(); // Submit the application - RMApp app = rm.submitApp(1024); + RMApp app = MockRMAppSubmitter.submitWithMemory(1024, rm); rm.drainEvents(); // Make a node to register so as to launch the AM. @@ -2496,7 +2497,7 @@ public void testUnregistrationOnlyIfRegistered() throws Exception { rm.start(); // Submit the application - RMApp rmApp = rm.submitApp(1024); + RMApp rmApp = MockRMAppSubmitter.submitWithMemory(1024, rm); rm.drainEvents(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 11264); @@ -2564,7 +2565,7 @@ public void testRMContainerAllocatorResendsRequestsOnRMRestart() rm1.start(); // Submit the application - RMApp app = rm1.submitApp(1024); + RMApp app = MockRMAppSubmitter.submitWithMemory(1024, rm1); rm1.drainEvents(); MockNM nm1 = new MockNM("h1:1234", 15120, rm1.getResourceTrackerService()); @@ -2817,7 +2818,7 @@ public void testRMUnavailable() MyResourceManager rm1 = new MyResourceManager(conf); rm1.start(); - RMApp app = rm1.submitApp(1024); + RMApp app = MockRMAppSubmitter.submitWithMemory(1024, rm1); rm1.drainEvents(); MockNM nm1 = new MockNM("h1:1234", 15120, rm1.getResourceTrackerService()); @@ -2869,7 +2870,7 @@ public void testAMRMTokenUpdate() throws Exception { rm.getRMContext().getAMRMTokenSecretManager(); // Submit the application - RMApp app = rm.submitApp(1024); + RMApp app = MockRMAppSubmitter.submitWithMemory(1024, rm); rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); @@ -3141,7 +3142,7 @@ public void testAttemptNotFoundCausesRMCommunicatorException() rm.start(); // Submit the application - RMApp app = rm.submitApp(1024); + RMApp app = MockRMAppSubmitter.submitWithMemory(1024, rm); rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); @@ -3175,7 +3176,7 @@ public void testUpdateAskOnRampDownAllReduces() throws Exception { rm.start(); // Submit the application - RMApp app = rm.submitApp(1024); + RMApp app = MockRMAppSubmitter.submitWithMemory(1024, rm); rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 1260); @@ -3346,7 +3347,7 @@ public void testAvoidAskMoreReducersWhenReducerPreemptionIsRequired() rm.start(); // Submit the application - RMApp app = rm.submitApp(1024); + RMApp app = MockRMAppSubmitter.submitWithMemory(1024, rm); rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 1260); @@ -3480,7 +3481,7 @@ public void testExcludeSchedReducesFromHeadroom() throws Exception { rm.start(); // Submit the application - RMApp app = rm.submitApp(1024); + RMApp app = MockRMAppSubmitter.submitWithMemory(1024, rm); rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 1260); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java index 51048660ef2..4adca73a306 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java @@ -58,6 +58,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore; @@ -127,7 +128,7 @@ public void testAMRMClientResendsRequestsOnRMRestart() throws Exception { rm1.start(); // Submit the application - RMApp app = rm1.submitApp(1024); + RMApp app = MockRMAppSubmitter.submitWithMemory(1024, rm1); rm1.drainEvents(); MockNM nm1 = new MockNM("h1:1234", 15120, rm1.getResourceTrackerService()); @@ -356,7 +357,7 @@ public void testAMRMClientForUnregisterAMOnRMRestart() throws Exception { rm1.start(); // Submit the application - RMApp app = rm1.submitApp(1024); + RMApp app = MockRMAppSubmitter.submitWithMemory(1024, rm1); rm1.drainEvents(); MockNM nm1 = new MockNM("h1:1234", 15120, rm1.getResourceTrackerService()); @@ -433,7 +434,7 @@ public void testAMRMClientOnAMRMTokenRollOverOnRMRestart() throws Exception { rm1.start(); Long startTime = System.currentTimeMillis(); // Submit the application - RMApp app = rm1.submitApp(1024); + RMApp app = MockRMAppSubmitter.submitWithMemory(1024, rm1); rm1.drainEvents(); MockNM nm1 = new MockNM("h1:1234", 15120, rm1.getResourceTrackerService()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java index 4af97e4212a..1142df45a8b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java @@ -68,6 +68,8 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.MiniYARNCluster; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.ParameterizedSchedulerTestBase; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; @@ -368,7 +370,7 @@ public void testSubmitIncorrectQueueToCapacityScheduler() throws IOException { public void testKillApplication() throws Exception { MockRM rm = new MockRM(); rm.start(); - RMApp app = rm.submitApp(2000); + RMApp app = MockRMAppSubmitter.submitWithMemory(2000, rm); @SuppressWarnings("resource") final YarnClient client = new MockYarnClient(); @@ -385,11 +387,19 @@ public void testApplicationType() throws Exception { GenericTestUtils.setRootLogLevel(Level.DEBUG); MockRM rm = new MockRM(); rm.start(); - RMApp app = rm.submitApp(2000); + RMApp app = MockRMAppSubmitter.submitWithMemory(2000, rm); RMApp app1 = - rm.submitApp(200, "name", "user", - new HashMap<>(), false, "default", -1, - null, "MAPREDUCE"); + MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm) + .withName("name") + .withUser("user") + .withAcls(new HashMap<>()) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(-1) + .withCredentials(null) + .withAppType("MAPREDUCE") + .build()); Assert.assertEquals("YARN", app.getApplicationType()); Assert.assertEquals("MAPREDUCE", app1.getApplicationType()); rm.stop(); @@ -401,9 +411,17 @@ public void testApplicationTypeLimit() throws Exception { MockRM rm = new MockRM(); rm.start(); RMApp app1 = - rm.submitApp(200, "name", "user", - new HashMap<>(), false, "default", -1, - null, "MAPREDUCE-LENGTH-IS-20"); + MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm) + .withName("name") + .withUser("user") + .withAcls(new HashMap<>()) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(-1) + .withCredentials(null) + .withAppType("MAPREDUCE-LENGTH-IS-20") + .build()); Assert.assertEquals("MAPREDUCE-LENGTH-IS-", app1.getApplicationType()); rm.stop(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterServiceTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterServiceTestBase.java index 80dbb8490e6..1020971de21 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterServiceTestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterServiceTestBase.java @@ -135,7 +135,7 @@ public void testRMIdentifierOnContainerAllocation() throws Exception { MockNM nm1 = rm.registerNode(DEFAULT_HOST + ":" + DEFAULT_PORT, 6 * GB); // Submit an application - RMApp app1 = rm.submitApp(2048); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(2048, rm); // kick the scheduling nm1.nodeHeartbeat(true); @@ -176,7 +176,7 @@ public void testAllocateResponseIdOverflow() throws Exception { MockNM nm1 = rm.registerNode(DEFAULT_HOST + ":" + DEFAULT_PORT, 6 * GB); // Submit an application - RMApp app1 = rm.submitApp(2048); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(2048, rm); // kick off the scheduling nm1.nodeHeartbeat(true); @@ -210,7 +210,7 @@ public void testInvalidContainerReleaseRequest() throws Exception { MockNM nm1 = rm.registerNode(DEFAULT_HOST + ":" + DEFAULT_PORT, 6 * GB); // Submit an application - RMApp app1 = rm.submitApp(1024); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(1024, rm); // kick the scheduling nm1.nodeHeartbeat(true); @@ -231,7 +231,7 @@ public void testInvalidContainerReleaseRequest() throws Exception { Assert.assertTrue(alloc1Response.getAllocatedContainers().size() > 0); - RMApp app2 = rm.submitApp(1024); + RMApp app2 = MockRMAppSubmitter.submitWithMemory(1024, rm); nm1.nodeHeartbeat(true); RMAppAttempt attempt2 = app2.getCurrentAppAttempt(); @@ -265,7 +265,7 @@ public void testProgressFilter() throws Exception{ MockNM nm1 = rm.registerNode(DEFAULT_HOST + ":" + DEFAULT_PORT, 6 * GB); // Submit an application - RMApp app1 = rm.submitApp(2048); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(2048, rm); nm1.nodeHeartbeat(true); RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); @@ -330,7 +330,7 @@ public void testFinishApplicationMasterBeforeRegistering() throws Exception { // Register node1 MockNM nm1 = rm.registerNode(DEFAULT_HOST + ":" + DEFAULT_PORT, 6 * GB); // Submit an application - RMApp app1 = rm.submitApp(2048); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(2048, rm); MockAM am1 = MockRM.launchAM(app1, rm, nm1); FinishApplicationMasterRequest req = FinishApplicationMasterRequest.newInstance( @@ -374,7 +374,8 @@ protected Dispatcher createDispatcher() { // Register node1 MockNM nm1 = rm.registerNode(DEFAULT_HOST + ":" + DEFAULT_PORT, 6 * GB); // Submit an application - RMApp app1 = rm.submitApp(2048); + RMApp app1 = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(2048, rm).build()); MockAM am1 = MockRM.launchAM(app1, rm, nm1); am1.registerAppAttempt(); FinishApplicationMasterRequest req = FinishApplicationMasterRequest @@ -449,7 +450,7 @@ public void testResourceTypes() throws Exception { MockRM rm = new MockRM(entry.getKey()); rm.start(); MockNM nm1 = rm.registerNode(DEFAULT_HOST + ":" + DEFAULT_PORT, 6 * GB); - RMApp app1 = rm.submitApp(2048); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(2048, rm); //Wait to make sure the attempt has the right state //TODO explore a better way than sleeping for a while (YARN-4929) Thread.sleep(1000); @@ -473,7 +474,7 @@ public void testAllocateAfterUnregister() throws Exception { MockNM nm1 = rm.registerNode(DEFAULT_HOST + ":" + DEFAULT_PORT, 6 * GB); // Submit an application - RMApp app1 = rm.submitApp(2048); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(2048, rm); nm1.nodeHeartbeat(true); RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); @@ -504,7 +505,7 @@ public void testUpdateTrackingUrl() throws Exception { // Register node1 MockNM nm1 = rm.registerNode(DEFAULT_HOST + ":" + DEFAULT_PORT, 6 * GB); - RMApp app1 = rm.submitApp(2048); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(2048, rm); nm1.nodeHeartbeat(true); RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); @@ -554,7 +555,15 @@ public void testValidateRequestCapacityAgainstMinMaxAllocation() .createResource(DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, null)); - RMApp app1 = rm.submitApp(GB, "app", "user", null, getDefaultQueueName()); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue(getDefaultQueueName()) + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); // Now request resource, memory > allowed @@ -611,7 +620,15 @@ public void testRequestCapacityMinMaxAllocationForResourceTypes() DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, ImmutableMap.of(CUSTOM_RES, 4))); - RMApp app1 = rm.submitApp(GB, "app", "user", null, getDefaultQueueName()); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue(getDefaultQueueName()) + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); Assert.assertEquals(Resource.newInstance(GB, 1), diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java index 9bbd148ff41..7f31d124f27 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java @@ -514,408 +514,6 @@ public GetNewApplicationResponse getNewAppId() throws Exception { .newRecord(GetNewApplicationRequest.class)); } - public RMApp submitApp(int masterMemory) throws Exception { - return submitApp(masterMemory, false); - } - - public RMApp submitApp(int masterMemory, String queue) throws Exception { - return submitApp(masterMemory, "", - UserGroupInformation.getCurrentUser().getShortUserName(), null, false, - queue, super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), - null); - } - - public RMApp submitApp(int masterMemory, Set appTags) - throws Exception { - return submitApp(masterMemory, null, false, null, Priority.newInstance(0), - appTags); - } - - public RMApp submitApp(int masterMemory, String queue, - boolean isAppIdProvided, ApplicationId appId, Priority priority, - Set appTags) throws Exception { - Resource resource = Resource.newInstance(masterMemory, 0); - ResourceRequest amResourceRequest = ResourceRequest.newInstance( - Priority.newInstance(0), ResourceRequest.ANY, resource, 1); - return submitApp(Collections.singletonList(amResourceRequest), "", - UserGroupInformation.getCurrentUser().getShortUserName(), null, false, - queue, super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, true, - false, isAppIdProvided, appId, 0, null, true, priority, null, - null, null, appTags); - } - - public RMApp submitApp(int masterMemory, Priority priority) throws Exception { - Resource resource = Resource.newInstance(masterMemory, 0); - return submitApp(resource, "", UserGroupInformation.getCurrentUser() - .getShortUserName(), null, false, null, - super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, true, - false, false, null, 0, null, true, priority); - } - - public RMApp submitApp(int masterMemory, Priority priority, - Credentials cred, ByteBuffer tokensConf) throws Exception { - Resource resource = Resource.newInstance(masterMemory, 0); - return submitApp(resource, "", UserGroupInformation.getCurrentUser() - .getShortUserName(), null, false, null, - super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), cred, null, true, - false, false, null, 0, null, true, priority, null, null, - tokensConf); - } - - public RMApp submitApp(int masterMemory, boolean unmanaged) - throws Exception { - return submitApp(masterMemory, "", UserGroupInformation.getCurrentUser() - .getShortUserName(), unmanaged); - } - - // client - public RMApp submitApp(int masterMemory, String name, String user) throws Exception { - return submitApp(masterMemory, name, user, false); - } - - public RMApp submitApp(int masterMemory, String name, String user, - boolean unmanaged) - throws Exception { - return submitApp(masterMemory, name, user, null, unmanaged, null, - super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null); - } - - public RMApp submitApp(int masterMemory, String name, String user, - Map acls) throws Exception { - return submitApp(masterMemory, name, user, acls, false, null, - super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null); - } - - public RMApp submitApp(int masterMemory, String name, String user, - Map acls, String queue) throws Exception { - return submitApp(masterMemory, name, user, acls, false, queue, - super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null); - } - - public RMApp submitApp(int masterMemory, String name, String user, - Map acls, String queue, String amLabel) - throws Exception { - Resource resource = Records.newRecord(Resource.class); - resource.setMemorySize(masterMemory); - Priority priority = Priority.newInstance(0); - return submitApp(resource, name, user, acls, false, queue, - super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, true, false, - false, null, 0, null, true, priority, amLabel, null, null); - } - - public RMApp submitApp(Resource resource, String name, String user, - Map acls, String queue) throws Exception { - return submitApp(resource, name, user, acls, false, queue, - super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, - true, false, false, null, 0, null, true, null); - } - - public RMApp submitApp(Resource resource, String name, String user, - Map acls, boolean unManaged, String queue) - throws Exception { - return submitApp(resource, name, user, acls, unManaged, queue, - super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, true, - false, false, null, 0, null, true, null); - } - - public RMApp submitApp(int masterMemory, String name, String user, - Map acls, String queue, - boolean waitForAccepted) throws Exception { - return submitApp(masterMemory, name, user, acls, false, queue, - super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, - waitForAccepted); - } - - public RMApp submitApp(int masterMemory, String name, String user, - Map acls, boolean unmanaged, String queue, - int maxAppAttempts, Credentials ts) throws Exception { - return submitApp(masterMemory, name, user, acls, unmanaged, queue, - maxAppAttempts, ts, null); - } - - public RMApp submitApp(int masterMemory, String name, String user, - Map acls, boolean unmanaged, String queue, - int maxAppAttempts, Credentials ts, String appType) throws Exception { - return submitApp(masterMemory, name, user, acls, unmanaged, queue, - maxAppAttempts, ts, appType, true); - } - - public RMApp submitApp(int masterMemory, String name, String user, - Map acls, boolean unmanaged, String queue, - int maxAppAttempts, Credentials ts, String appType, - boolean waitForAccepted) - throws Exception { - return submitApp(masterMemory, name, user, acls, unmanaged, queue, - maxAppAttempts, ts, appType, waitForAccepted, false); - } - - public RMApp submitApp(int masterMemory, String name, String user, - Map acls, boolean unmanaged, String queue, - int maxAppAttempts, Credentials ts, String appType, - boolean waitForAccepted, boolean keepContainers) throws Exception { - Resource resource = Records.newRecord(Resource.class); - resource.setMemorySize(masterMemory); - return submitApp(resource, name, user, acls, unmanaged, queue, - maxAppAttempts, ts, appType, waitForAccepted, keepContainers, - false, null, 0, null, true, Priority.newInstance(0)); - } - - public RMApp submitApp(int masterMemory, long attemptFailuresValidityInterval, - boolean keepContainers) throws Exception { - Resource resource = Records.newRecord(Resource.class); - resource.setMemorySize(masterMemory); - Priority priority = Priority.newInstance(0); - return submitApp(resource, "", UserGroupInformation.getCurrentUser() - .getShortUserName(), null, false, null, - super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, true, keepContainers, - false, null, attemptFailuresValidityInterval, null, true, priority); - } - - public RMApp submitApp(int masterMemory, String name, String user, - Map acls, boolean unmanaged, String queue, - int maxAppAttempts, Credentials ts, String appType, - boolean waitForAccepted, boolean keepContainers, boolean isAppIdProvided, - ApplicationId applicationId) throws Exception { - Resource resource = Records.newRecord(Resource.class); - resource.setMemorySize(masterMemory); - Priority priority = Priority.newInstance(0); - return submitApp(resource, name, user, acls, unmanaged, queue, - maxAppAttempts, ts, appType, waitForAccepted, keepContainers, - isAppIdProvided, applicationId, 0, null, true, priority); - } - - public RMApp submitApp(int masterMemory, - LogAggregationContext logAggregationContext) throws Exception { - Resource resource = Records.newRecord(Resource.class); - resource.setMemorySize(masterMemory); - Priority priority = Priority.newInstance(0); - return submitApp(resource, "", UserGroupInformation.getCurrentUser() - .getShortUserName(), null, false, null, - super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, true, false, - false, null, 0, logAggregationContext, true, priority); - } - - public RMApp submitApp(Resource capability, String name, String user, - Map acls, boolean unmanaged, String queue, - int maxAppAttempts, Credentials ts, String appType, - boolean waitForAccepted, boolean keepContainers, boolean isAppIdProvided, - ApplicationId applicationId, long attemptFailuresValidityInterval, - LogAggregationContext logAggregationContext, - boolean cancelTokensWhenComplete, Priority priority) throws Exception { - return submitApp(capability, name, user, acls, unmanaged, queue, - maxAppAttempts, ts, appType, waitForAccepted, keepContainers, - isAppIdProvided, applicationId, attemptFailuresValidityInterval, - logAggregationContext, cancelTokensWhenComplete, priority, "", null, - null); - } - - public RMApp submitApp(Credentials cred, ByteBuffer tokensConf) - throws Exception { - return submitApp(Resource.newInstance(200, 1), "app1", "user", null, false, - null, super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), cred, null, true, - false, false, null, 0, null, true, Priority.newInstance(0), null, null, - tokensConf); - } - - public RMApp submitApp(List amResourceRequests) - throws Exception { - return submitApp(amResourceRequests, "app1", - "user", null, false, null, - super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, true, - false, false, null, 0, null, true, - amResourceRequests.get(0).getPriority(), - amResourceRequests.get(0).getNodeLabelExpression(), null, null); - } - - public RMApp submitApp(List amResourceRequests, - String appNodeLabel) throws Exception { - return submitApp(amResourceRequests, "app1", "user", null, false, null, - super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, true, - false, false, null, 0, null, true, - amResourceRequests.get(0).getPriority(), - amResourceRequests.get(0).getNodeLabelExpression(), null, null, null, - appNodeLabel); - } - - public RMApp submitApp(Resource capability, String name, String user, - Map acls, boolean unmanaged, String queue, - int maxAppAttempts, Credentials ts, String appType, - boolean waitForAccepted, boolean keepContainers, boolean isAppIdProvided, - ApplicationId applicationId, long attemptFailuresValidityInterval, - LogAggregationContext logAggregationContext, - boolean cancelTokensWhenComplete, Priority priority, String amLabel, - Map applicationTimeouts, - ByteBuffer tokensConf) - throws Exception { - priority = (priority == null) ? Priority.newInstance(0) : priority; - ResourceRequest amResourceRequest = ResourceRequest.newInstance( - priority, ResourceRequest.ANY, capability, 1); - if (amLabel != null && !amLabel.isEmpty()) { - amResourceRequest.setNodeLabelExpression(amLabel.trim()); - } - return submitApp(Collections.singletonList(amResourceRequest), name, user, - acls, unmanaged, queue, maxAppAttempts, ts, appType, waitForAccepted, - keepContainers, isAppIdProvided, applicationId, - attemptFailuresValidityInterval, logAggregationContext, - cancelTokensWhenComplete, priority, amLabel, applicationTimeouts, - tokensConf); - } - - public RMApp submitApp(List amResourceRequests, String name, - String user, Map acls, boolean unmanaged, - String queue, int maxAppAttempts, Credentials ts, String appType, - boolean waitForAccepted, boolean keepContainers, boolean isAppIdProvided, - ApplicationId applicationId, long attemptFailuresValidityInterval, - LogAggregationContext logAggregationContext, - boolean cancelTokensWhenComplete, Priority priority, String amLabel, - Map applicationTimeouts, - ByteBuffer tokensConf) throws Exception { - return submitApp(amResourceRequests, name, user, acls, unmanaged, queue, - maxAppAttempts, ts, appType, waitForAccepted, keepContainers, - isAppIdProvided, applicationId, attemptFailuresValidityInterval, - logAggregationContext, cancelTokensWhenComplete, priority, amLabel, - applicationTimeouts, tokensConf, null); - } - - public RMApp submitApp(List amResourceRequests, String name, - String user, Map acls, boolean unmanaged, - String queue, int maxAppAttempts, Credentials ts, String appType, - boolean waitForAccepted, boolean keepContainers, boolean isAppIdProvided, - ApplicationId applicationId, long attemptFailuresValidityInterval, - LogAggregationContext logAggregationContext, - boolean cancelTokensWhenComplete, Priority priority, String amLabel, - Map applicationTimeouts, - ByteBuffer tokensConf, Set applicationTags) throws Exception { - return submitApp(amResourceRequests, name, user, acls, unmanaged, queue, - maxAppAttempts, ts, appType, waitForAccepted, keepContainers, - isAppIdProvided, applicationId, attemptFailuresValidityInterval, - logAggregationContext, cancelTokensWhenComplete, priority, amLabel, - applicationTimeouts, tokensConf, applicationTags, null); - } - - public RMApp submitApp(List amResourceRequests, String name, - String user, Map acls, boolean unmanaged, - String queue, int maxAppAttempts, Credentials ts, String appType, - boolean waitForAccepted, boolean keepContainers, boolean isAppIdProvided, - ApplicationId applicationId, long attemptFailuresValidityInterval, - LogAggregationContext logAggregationContext, - boolean cancelTokensWhenComplete, Priority priority, String amLabel, - Map applicationTimeouts, - ByteBuffer tokensConf, Set applicationTags, String appNodeLabel) - throws Exception { - ApplicationId appId = isAppIdProvided ? applicationId : null; - ApplicationClientProtocol client = getClientRMService(); - if (! isAppIdProvided) { - GetNewApplicationResponse resp = client.getNewApplication(Records - .newRecord(GetNewApplicationRequest.class)); - appId = resp.getApplicationId(); - } - SubmitApplicationRequest req = Records - .newRecord(SubmitApplicationRequest.class); - ApplicationSubmissionContext sub = Records - .newRecord(ApplicationSubmissionContext.class); - sub.setKeepContainersAcrossApplicationAttempts(keepContainers); - sub.setApplicationId(appId); - sub.setApplicationName(name); - sub.setMaxAppAttempts(maxAppAttempts); - if (applicationTags != null) { - sub.setApplicationTags(applicationTags); - } - if (applicationTimeouts != null && applicationTimeouts.size() > 0) { - sub.setApplicationTimeouts(applicationTimeouts); - } - if (unmanaged) { - sub.setUnmanagedAM(true); - } - if (queue != null) { - sub.setQueue(queue); - } - if (priority != null) { - sub.setPriority(priority); - } - if (appNodeLabel != null) { - sub.setNodeLabelExpression(appNodeLabel); - } - sub.setApplicationType(appType); - ContainerLaunchContext clc = Records - .newRecord(ContainerLaunchContext.class); - clc.setApplicationACLs(acls); - if (ts != null && UserGroupInformation.isSecurityEnabled()) { - DataOutputBuffer dob = new DataOutputBuffer(); - ts.writeTokenStorageToStream(dob); - ByteBuffer securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); - clc.setTokens(securityTokens); - clc.setTokensConf(tokensConf); - } - sub.setAMContainerSpec(clc); - sub.setAttemptFailuresValidityInterval(attemptFailuresValidityInterval); - if (logAggregationContext != null) { - sub.setLogAggregationContext(logAggregationContext); - } - sub.setCancelTokensWhenComplete(cancelTokensWhenComplete); - if (amLabel != null && !amLabel.isEmpty()) { - for (ResourceRequest amResourceRequest : amResourceRequests) { - amResourceRequest.setNodeLabelExpression(amLabel.trim()); - } - } - sub.setAMContainerResourceRequests(amResourceRequests); - req.setApplicationSubmissionContext(sub); - UserGroupInformation fakeUser = - UserGroupInformation.createUserForTesting(user, new String[] {"someGroup"}); - PrivilegedExceptionAction action = - new PrivilegedExceptionAction() { - ApplicationClientProtocol client; - SubmitApplicationRequest req; - @Override - public SubmitApplicationResponse run() throws IOException, YarnException { - try { - return client.submitApplication(req); - } catch (YarnException | IOException e) { - e.printStackTrace(); - throw e; - } - } - PrivilegedExceptionAction setClientReq( - ApplicationClientProtocol client, SubmitApplicationRequest req) { - this.client = client; - this.req = req; - return this; - } - }.setClientReq(client, req); - fakeUser.doAs(action); - // make sure app is immediately available after submit - if (waitForAccepted) { - waitForState(appId, RMAppState.ACCEPTED); - } - RMApp rmApp = getRMContext().getRMApps().get(appId); - - // unmanaged AM won't go to RMAppAttemptState.SCHEDULED. - if (waitForAccepted && !unmanaged) { - waitForState(rmApp.getCurrentAppAttempt().getAppAttemptId(), - RMAppAttemptState.SCHEDULED); - } - - ((AbstractYarnScheduler)getResourceScheduler()).update(); - - return rmApp; - } - public MockNM unRegisterNode(MockNM nm) throws Exception { nm.unRegisterNode(); drainEventsImplicitly(); @@ -1423,18 +1021,7 @@ public void disableDrainEventsImplicitly() { public void enableDrainEventsImplicityly() { disableDrainEventsImplicitly = false; } - - public RMApp submitApp(int masterMemory, Priority priority, - Map applicationTimeouts) throws Exception { - Resource resource = Resource.newInstance(masterMemory, 0); - return submitApp( - resource, "", UserGroupInformation.getCurrentUser().getShortUserName(), - null, false, null, - super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, true, - false, false, null, 0, null, true, priority, null, applicationTimeouts, - null); - } + @Override protected void serviceInit(Configuration conf) throws Exception { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRMAppSubmissionData.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRMAppSubmissionData.java new file mode 100644 index 00000000000..69cff7e67c3 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRMAppSubmissionData.java @@ -0,0 +1,368 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager; + +import org.apache.hadoop.security.Credentials; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.api.records.ApplicationAccessType; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType; +import org.apache.hadoop.yarn.api.records.LogAggregationContext; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.util.Records; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class MockRMAppSubmissionData { + private final List amResourceRequests; + private final String name; + private final String user; + private final Map acls; + private final boolean unmanaged; + private final String queue; + private final int maxAppAttempts; + private final String appType; + private final boolean waitForAccepted; + private final boolean keepContainers; + private final boolean isAppIdProvided; + private final ApplicationId applicationId; + private final long attemptFailuresValidityInterval; + private final LogAggregationContext logAggregationContext; + private final boolean cancelTokensWhenComplete; + private final Priority priority; + private final String amLabel; + private final Map applicationTimeouts; + private final ByteBuffer tokensConf; + private final Set applicationTags; + private final String appNodeLabel; + private final Credentials credentials; + private final Resource resource; + + public List getAmResourceRequests() { + return amResourceRequests; + } + + public String getName() { + return name; + } + + public String getUser() { + return user; + } + + public Map getAcls() { + return acls; + } + + public boolean isUnmanaged() { + return unmanaged; + } + + public String getQueue() { + return queue; + } + + public int getMaxAppAttempts() { + return maxAppAttempts; + } + + public String getAppType() { + return appType; + } + + public boolean isWaitForAccepted() { + return waitForAccepted; + } + + public boolean isKeepContainers() { + return keepContainers; + } + + public boolean isAppIdProvided() { + return isAppIdProvided; + } + + public ApplicationId getApplicationId() { + return applicationId; + } + + public long getAttemptFailuresValidityInterval() { + return attemptFailuresValidityInterval; + } + + public LogAggregationContext getLogAggregationContext() { + return logAggregationContext; + } + + public boolean isCancelTokensWhenComplete() { + return cancelTokensWhenComplete; + } + + public Priority getPriority() { + return priority; + } + + public String getAmLabel() { + return amLabel; + } + + public Map getApplicationTimeouts() { + return applicationTimeouts; + } + + public ByteBuffer getTokensConf() { + return tokensConf; + } + + public Set getApplicationTags() { + return applicationTags; + } + + public String getAppNodeLabel() { + return appNodeLabel; + } + + public Credentials getCredentials() { + return credentials; + } + + public Resource getResource() { + return resource; + } + + private MockRMAppSubmissionData(Builder builder) { + this.amLabel = builder.amLabel; + this.tokensConf = builder.tokensConf; + this.maxAppAttempts = builder.maxAppAttempts; + this.logAggregationContext = builder.logAggregationContext; + this.queue = builder.queue; + this.amResourceRequests = builder.amResourceRequests; + this.user = builder.user; + this.priority = builder.priority; + this.waitForAccepted = builder.waitForAccepted; + this.keepContainers = builder.keepContainers; + this.name = builder.name; + this.applicationId = builder.applicationId; + this.attemptFailuresValidityInterval = builder.attemptFailuresValidityInterval; + this.acls = builder.acls; + this.appType = builder.appType; + this.appNodeLabel = builder.appNodeLabel; + this.isAppIdProvided = builder.isAppIdProvided; + this.unmanaged = builder.unmanaged; + this.applicationTags = builder.applicationTags; + this.cancelTokensWhenComplete = builder.cancelTokensWhenComplete; + this.applicationTimeouts = builder.applicationTimeouts; + this.credentials = builder.credentials; + this.resource = builder.resource; + } + + public static final class Builder { + private List amResourceRequests; + private String name; + private String user; + private Map acls; + private boolean unmanaged; + private String queue; + private int maxAppAttempts; + private String appType; + private boolean waitForAccepted; + private boolean keepContainers; + private boolean isAppIdProvided; + private ApplicationId applicationId; + private long attemptFailuresValidityInterval; + private LogAggregationContext logAggregationContext; + private boolean cancelTokensWhenComplete; + private Priority priority; + private String amLabel; + private Map applicationTimeouts; + private ByteBuffer tokensConf; + private Set applicationTags; + private String appNodeLabel; + private Credentials credentials; + private Resource resource; + + private Builder() { + } + + public static Builder create() { + return new Builder(); + } + + public static Builder createWithDefaults(long memory, MockRM mockRM) throws IOException { + Resource resource = Records.newRecord(Resource.class); + resource.setMemorySize(memory); + return createWithDefaults(resource, mockRM); + } + +// public static Builder createWithDefaults(Resource resource, MockRM mockRM) throws IOException { +// return createWithDefaultsInternal(resource, mockRM); +// } + + public static Builder createWithDefaults(Resource resource, MockRM mockRM) throws IOException { + int maxAppAttempts = mockRM.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, + YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS); + return MockRMAppSubmissionData.Builder.create() + .withResource(resource) + .withName("") + .withUser(UserGroupInformation + .getCurrentUser().getShortUserName()) + .withAcls(null) + .withUnmanaged(false) + .withQueue(null) + .withMaxAppAttempts(maxAppAttempts) + .withCredentials(null) + .withAppType(null) + .withWaitForAccepted(true) + .withKeepContainers(false) + .withIsAppIdProvided(false) + .withApplicationId(null) + .withAttemptFailuresValidityInterval(0L) + .withLogAggregationContext(null) + .withCancelTokensWhenComplete(true) + .withPriority(Priority.newInstance(0)) + .withAmLabel("") + .withApplicationTimeouts(null) + .withTokensConf(null); + } + + public Builder withAmResourceRequests(List amResourceRequests) { + this.amResourceRequests = amResourceRequests; + return this; + } + + public Builder withName(String name) { + this.name = name; + return this; + } + + public Builder withUser(String user) { + this.user = user; + return this; + } + + public Builder withAcls(Map acls) { + this.acls = acls; + return this; + } + + public Builder withUnmanaged(boolean unmanaged) { + this.unmanaged = unmanaged; + return this; + } + + public Builder withQueue(String queue) { + this.queue = queue; + return this; + } + + public Builder withMaxAppAttempts(int maxAppAttempts) { + this.maxAppAttempts = maxAppAttempts; + return this; + } + + public Builder withAppType(String appType) { + this.appType = appType; + return this; + } + + public Builder withWaitForAccepted(boolean waitForAccepted) { + this.waitForAccepted = waitForAccepted; + return this; + } + + public Builder withKeepContainers(boolean keepContainers) { + this.keepContainers = keepContainers; + return this; + } + + public Builder withIsAppIdProvided(boolean isAppIdProvided) { + this.isAppIdProvided = isAppIdProvided; + return this; + } + + public Builder withApplicationId(ApplicationId applicationId) { + this.applicationId = applicationId; + return this; + } + + public Builder withAttemptFailuresValidityInterval(long attemptFailuresValidityInterval) { + this.attemptFailuresValidityInterval = attemptFailuresValidityInterval; + return this; + } + + public Builder withLogAggregationContext(LogAggregationContext logAggregationContext) { + this.logAggregationContext = logAggregationContext; + return this; + } + + public Builder withCancelTokensWhenComplete(boolean cancelTokensWhenComplete) { + this.cancelTokensWhenComplete = cancelTokensWhenComplete; + return this; + } + + public Builder withPriority(Priority priority) { + this.priority = priority; + return this; + } + + public Builder withAmLabel(String amLabel) { + this.amLabel = amLabel; + return this; + } + + public Builder withApplicationTimeouts(Map applicationTimeouts) { + this.applicationTimeouts = applicationTimeouts; + return this; + } + + public Builder withTokensConf(ByteBuffer tokensConf) { + this.tokensConf = tokensConf; + return this; + } + + public Builder withApplicationTags(Set applicationTags) { + this.applicationTags = applicationTags; + return this; + } + + public Builder withAppNodeLabel(String appNodeLabel) { + this.appNodeLabel = appNodeLabel; + return this; + } + + public Builder withCredentials(Credentials cred) { + this.credentials = cred; + return this; + } + + public Builder withResource(Resource resource) { + this.resource = resource; + return this; + } + + public MockRMAppSubmissionData build() { + return new MockRMAppSubmissionData(this); + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRMAppSubmitter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRMAppSubmitter.java new file mode 100644 index 00000000000..d70ee048fee --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRMAppSubmitter.java @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.security.PrivilegedExceptionAction; +import java.util.Collections; +import java.util.List; + +import org.apache.commons.lang3.builder.ReflectionToStringBuilder; +import org.apache.hadoop.io.DataOutputBuffer; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; +import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler; +import org.apache.hadoop.yarn.util.Records; +import org.junit.Rule; +import org.junit.rules.TestRule; +import org.junit.rules.TestWatcher; +import org.junit.runner.Description; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MockRMAppSubmitter { + private static final Logger LOG = LoggerFactory.getLogger(MockRM.class); + + public static RMApp submitWithMemory(long memory, MockRM mockRM) throws Exception { + Resource resource = Records.newRecord(Resource.class); + resource.setMemorySize(memory); + MockRMAppSubmissionData data = MockRMAppSubmissionData.Builder.createWithDefaults(resource, mockRM) + .build(); + return MockRMAppSubmitter.submit(mockRM, data); + } + + public static RMApp submit(MockRM mockRM, MockRMAppSubmissionData data) throws Exception { + ApplicationId appId = data.isAppIdProvided() ? data.getApplicationId() : null; + ApplicationClientProtocol client = mockRM.getClientRMService(); + if (!data.isAppIdProvided()) { + GetNewApplicationResponse resp = client.getNewApplication(Records + .newRecord(GetNewApplicationRequest.class)); + appId = resp.getApplicationId(); + } + SubmitApplicationRequest req = Records + .newRecord(SubmitApplicationRequest.class); + ApplicationSubmissionContext sub = Records + .newRecord(ApplicationSubmissionContext.class); + sub.setKeepContainersAcrossApplicationAttempts(data.isKeepContainers()); + sub.setApplicationId(appId); + sub.setApplicationName(data.getName()); + sub.setMaxAppAttempts(data.getMaxAppAttempts()); + if (data.getApplicationTags() != null) { + sub.setApplicationTags(data.getApplicationTags()); + } + if (data.getApplicationTimeouts() != null && data.getApplicationTimeouts().size() > 0) { + sub.setApplicationTimeouts(data.getApplicationTimeouts()); + } + if (data.isUnmanaged()) { + sub.setUnmanagedAM(true); + } + if (data.getQueue() != null) { + sub.setQueue(data.getQueue()); + } + if (data.getPriority() != null) { + sub.setPriority(data.getPriority()); + } + if (data.getAppNodeLabel() != null) { + sub.setNodeLabelExpression(data.getAppNodeLabel()); + } + sub.setApplicationType(data.getAppType()); + ContainerLaunchContext clc = Records + .newRecord(ContainerLaunchContext.class); + clc.setApplicationACLs(data.getAcls()); + if (data.getCredentials() != null && UserGroupInformation.isSecurityEnabled()) { + DataOutputBuffer dob = new DataOutputBuffer(); + data.getCredentials().writeTokenStorageToStream(dob); + ByteBuffer securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); + clc.setTokens(securityTokens); + clc.setTokensConf(data.getTokensConf()); + } + sub.setAMContainerSpec(clc); + sub.setAttemptFailuresValidityInterval(data.getAttemptFailuresValidityInterval()); + if (data.getLogAggregationContext() != null) { + sub.setLogAggregationContext(data.getLogAggregationContext()); + } + sub.setCancelTokensWhenComplete(data.isCancelTokensWhenComplete()); + + Priority priority = data.getPriority(); + if (priority == null) { + priority = Priority.newInstance(0); + } + + List amResourceRequests = data.getAmResourceRequests(); + if (amResourceRequests == null || amResourceRequests.isEmpty()) { + ResourceRequest amResReq = ResourceRequest.newInstance( + priority, ResourceRequest.ANY, data.getResource(), 1); + amResourceRequests = Collections.singletonList(amResReq); + } + + if (data.getAmLabel() != null && !data.getAmLabel().isEmpty()) { + for (ResourceRequest amResourceRequest : amResourceRequests) { + amResourceRequest.setNodeLabelExpression(data.getAmLabel().trim()); + } + } + sub.setAMContainerResourceRequests(amResourceRequests); + + req.setApplicationSubmissionContext(sub); + UserGroupInformation fakeUser = + UserGroupInformation.createUserForTesting(data.getUser(), new String[] {"someGroup"}); + PrivilegedExceptionAction action = + new PrivilegedExceptionAction() { + ApplicationClientProtocol client; + SubmitApplicationRequest req; + @Override + public SubmitApplicationResponse run() throws IOException, YarnException { + try { + return client.submitApplication(req); + } catch (YarnException | IOException e) { + e.printStackTrace(); + throw e; + } + } + PrivilegedExceptionAction setClientReq( + ApplicationClientProtocol client, SubmitApplicationRequest req) { + this.client = client; + this.req = req; + return this; + } + }.setClientReq(client, req); + fakeUser.doAs(action); + // make sure app is immediately available after submit + if (data.isWaitForAccepted()) { + mockRM.waitForState(appId, RMAppState.ACCEPTED); + } + RMApp rmApp = mockRM.getRMContext().getRMApps().get(appId); + + // unmanaged AM won't go to RMAppAttemptState.SCHEDULED. + if (data.isWaitForAccepted() && !data.isUnmanaged()) { + mockRM.waitForState(rmApp.getCurrentAppAttempt().getAppAttemptId(), + RMAppAttemptState.SCHEDULED); + } + + ((AbstractYarnScheduler)mockRM.getResourceScheduler()).update(); + + return rmApp; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java index fcd4ed1f444..0d4cc12b9ce 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java @@ -270,7 +270,13 @@ public void testAuthorizedAccess() throws Exception { Map acls = new HashMap(2); acls.put(ApplicationAccessType.VIEW_APP, "*"); - RMApp app = rm.submitApp(1024, "appname", "appuser", acls); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm) + .withName("appname") + .withUser("appuser") + .withAcls(acls) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); nm1.nodeHeartbeat(true); @@ -328,7 +334,7 @@ public void testUnauthorizedAccess() throws Exception { MockNM nm1 = rm.registerNode("localhost:1234", 5120); - RMApp app = rm.submitApp(1024); + RMApp app = MockRMAppSubmitter.submitWithMemory(1024, rm); nm1.nodeHeartbeat(true); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationCleanup.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationCleanup.java index 373fa90325b..807000a9810 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationCleanup.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationCleanup.java @@ -81,7 +81,7 @@ public void testAppCleanup() throws Exception { MockNM nm1 = rm.registerNode("127.0.0.1:1234", 5000); - RMApp app = rm.submitApp(2000); + RMApp app = MockRMAppSubmitter.submitWithMemory(2000, rm); //kick the scheduling nm1.nodeHeartbeat(true); @@ -159,7 +159,7 @@ public void testContainerCleanup() throws Exception { MockNM nm1 = rm.registerNode("127.0.0.1:1234", 5000); - RMApp app = rm.submitApp(2000); + RMApp app = MockRMAppSubmitter.submitWithMemory(2000, rm); //kick the scheduling nm1.nodeHeartbeat(true); @@ -296,7 +296,7 @@ public void testAppCleanupWhenRMRestartedAfterAppFinished() throws Exception { nm1.registerNode(); // create app and launch the AM - RMApp app0 = rm1.submitApp(200); + RMApp app0 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am0 = launchAM(app0, rm1, nm1); nm1.nodeHeartbeat(am0.getApplicationAttemptId(), 1, ContainerState.COMPLETE); rm1.waitForState(app0.getApplicationId(), RMAppState.FAILED); @@ -333,7 +333,7 @@ public void testAppCleanupWhenRMRestartedBeforeAppFinished() throws Exception { nm2.registerNode(); // create app and launch the AM - RMApp app0 = rm1.submitApp(200); + RMApp app0 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am0 = launchAM(app0, rm1, nm1); // alloc another container on nm2 @@ -387,7 +387,7 @@ public void testContainerCleanupWhenRMRestartedAppNotRegistered() throws nm1.registerNode(); // create app and launch the AM - RMApp app0 = rm1.submitApp(200); + RMApp app0 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am0 = launchAM(app0, rm1, nm1); nm1.nodeHeartbeat(am0.getApplicationAttemptId(), 1, ContainerState.RUNNING); rm1.waitForState(app0.getApplicationId(), RMAppState.RUNNING); @@ -423,7 +423,7 @@ public void testAppCleanupWhenNMReconnects() throws Exception { nm1.registerNode(); // create app and launch the AM - RMApp app0 = rm1.submitApp(200); + RMApp app0 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am0 = launchAM(app0, rm1, nm1); nm1.nodeHeartbeat(am0.getApplicationAttemptId(), 1, ContainerState.COMPLETE); rm1.waitForState(app0.getApplicationId(), RMAppState.FAILED); @@ -463,7 +463,7 @@ public void testProcessingNMContainerStatusesOnNMRestart() throws Exception { new MockNM("127.0.0.1:1234", nmMemory, rm1.getResourceTrackerService()); nm1.registerNode(); - RMApp app0 = rm1.submitApp(amMemory); + RMApp app0 = MockRMAppSubmitter.submitWithMemory(amMemory, rm1); MockAM am0 = MockRM.launchAndRegisterAM(app0, rm1, nm1); // 2. AM sends ResourceRequest for 1 container with memory 2048MB. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java index 55b4935ab14..3cf809d7f3e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java @@ -230,7 +230,7 @@ public void testAMLaunchAndCleanup() throws Exception { rm.start(); MockNM nm1 = rm.registerNode("127.0.0.1:1234", 5120); - RMApp app = rm.submitApp(2000); + RMApp app = MockRMAppSubmitter.submitWithMemory(2000, rm); // kick the scheduling nm1.nodeHeartbeat(true); @@ -289,7 +289,7 @@ public void testAMCleanupBeforeLaunch() throws Exception { MockRM rm = new MockRM(); rm.start(); MockNM nm1 = rm.registerNode("127.0.0.1:1234", 5120); - RMApp app = rm.submitApp(2000); + RMApp app = MockRMAppSubmitter.submitWithMemory(2000, rm); // kick the scheduling nm1.nodeHeartbeat(true); RMAppAttempt attempt = app.getCurrentAppAttempt(); @@ -365,7 +365,7 @@ protected YarnRPC getYarnRPC() { rm.start(); MockNM nm1 = rm.registerNode("127.0.0.1:1234", 5120); - RMApp app = rm.submitApp(2000); + RMApp app = MockRMAppSubmitter.submitWithMemory(2000, rm); // kick the scheduling nm1.nodeHeartbeat(true); @@ -385,7 +385,7 @@ public void testallocateBeforeAMRegistration() throws Exception { MockRM rm = new MockRM(); rm.start(); MockNM nm1 = rm.registerNode("h1:1234", 5000); - RMApp app = rm.submitApp(2000); + RMApp app = MockRMAppSubmitter.submitWithMemory(2000, rm); // kick the scheduling nm1.nodeHeartbeat(true); RMAppAttempt attempt = app.getCurrentAppAttempt(); @@ -458,7 +458,7 @@ private void testSetupTokens(boolean https, YarnConfiguration conf) MockRM rm = new MockRM(conf); rm.start(); MockNM nm1 = rm.registerNode("h1:1234", 5000); - RMApp app = rm.submitApp(2000); + RMApp app = MockRMAppSubmitter.submitWithMemory(2000, rm); /// kick the scheduling nm1.nodeHeartbeat(true); RMAppAttempt attempt = app.getCurrentAppAttempt(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterServiceCapacity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterServiceCapacity.java index 8d04e5061c2..e99427320a7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterServiceCapacity.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterServiceCapacity.java @@ -105,7 +105,7 @@ public void testInvalidIncreaseDecreaseRequest() throws Exception { MockNM nm1 = rm.registerNode(DEFAULT_HOST + ":" + DEFAULT_PORT, 6 * GB); // Submit an application - RMApp app1 = rm.submitApp(1024); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(1024, rm); // kick the scheduling nm1.nodeHeartbeat(true); @@ -177,7 +177,10 @@ public void testPriorityInAllocatedResponse() throws Exception { // Submit an application Priority appPriority1 = Priority.newInstance(5); - RMApp app1 = rm.submitApp(2048, appPriority1); + MockRMAppSubmissionData data = MockRMAppSubmissionData.Builder.createWithDefaults(2048, rm) + .withPriority(appPriority1) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); nm1.nodeHeartbeat(true); RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterServiceFair.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterServiceFair.java index 3ff3bcd1224..6b57ca189ab 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterServiceFair.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterServiceFair.java @@ -97,7 +97,15 @@ public void testRequestCapacityMinMaxAllocationWithDifferentUnits() ImmutableMap. builder() .put(CUSTOM_RES, "5G").build())); - RMApp app1 = rm.submitApp(GB, "app", "user", null, DEFAULT_QUEUE); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue(DEFAULT_QUEUE) + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); // Now request res_1, 500M < 5G so it should be allowed diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterServiceInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterServiceInterceptor.java index 7da4d979a43..e1d6defa63d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterServiceInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterServiceInterceptor.java @@ -171,7 +171,7 @@ public void testApplicationMasterInterceptor() throws Exception { MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * GB); // Submit an application - RMApp app1 = rm.submitApp(2048); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(2048, rm); // kick the scheduling nm1.nodeHeartbeat(true); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestCapacitySchedulerMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestCapacitySchedulerMetrics.java index 572619ff632..d410d744774 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestCapacitySchedulerMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestCapacitySchedulerMetrics.java @@ -78,8 +78,18 @@ public RMNodeLabelsManager createNodeLabelManager() { Assert.assertEquals(0, csMetrics.getNumOfAllocates()); Assert.assertEquals(0, csMetrics.getNumOfCommitSuccess()); - RMApp rmApp = rm.submitApp(1024, "app", "user", null, false, - "default", 1, null, null, false); + RMApp rmApp = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(1) + .withCredentials(null) + .withAppType(null) + .withWaitForAccepted(false) + .build()); MockAM am = MockRM.launchAMWhenAsyncSchedulingEnabled(rmApp, rm); am.registerAppAttempt(); am.allocate("*", 1024, 1, new ArrayList<>()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index d422233a0d1..4854b03ce02 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -605,8 +605,12 @@ public void testForceKillApplication() throws Exception { GetApplicationsRequest getRequest = GetApplicationsRequest.newInstance( EnumSet.of(YarnApplicationState.KILLED)); - RMApp app1 = rm.submitApp(1024); - RMApp app2 = rm.submitApp(1024, true); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(1024, rm); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm) + .withUnmanaged(true) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm, data); assertEquals("Incorrect number of apps in the RM", 0, rmService.getApplications(getRequest).getApplicationList().size()); @@ -2437,7 +2441,10 @@ public void testUpdatePriorityAndKillAppWithZeroClusterResource() MockRM rm = new MockRM(conf); rm.init(conf); rm.start(); - RMApp app1 = rm.submitApp(1024, Priority.newInstance(appPriority)); + MockRMAppSubmissionData data = MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm) + .withPriority(Priority.newInstance(appPriority)) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); ClientRMService rmService = rm.getClientRMService(); testApplicationPriorityUpdation(rmService, app1, appPriority, appPriority); rm.killApp(app1.getApplicationId()); @@ -2460,7 +2467,10 @@ public void testUpdateApplicationPriorityRequest() throws Exception { rm.start(); rm.registerNode("host1:1234", 1024); // Start app1 with appPriority 5 - RMApp app1 = rm.submitApp(1024, Priority.newInstance(appPriority)); + MockRMAppSubmissionData data = MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm) + .withPriority(Priority.newInstance(appPriority)) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); Assert.assertEquals("Incorrect priority has been set to application", appPriority, app1.getApplicationPriority().getPriority()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java index a9093f99d6b..ee7b92a5cf5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java @@ -78,7 +78,7 @@ public void testUsageWithOneAttemptAndOneContainer() throws Exception { new MockNM("127.0.0.1:1234", 15120, rm.getResourceTrackerService()); nm.registerNode(); - RMApp app0 = rm.submitApp(200); + RMApp app0 = MockRMAppSubmitter.submitWithMemory(200, rm); RMAppMetrics rmAppMetrics = app0.getRMAppMetrics(); Assert.assertTrue( @@ -147,7 +147,7 @@ public void testUsageWithMultipleContainersAndRMRestart() throws Exception { new MockNM("127.0.0.1:1234", 65536, rm0.getResourceTrackerService()); nm.registerNode(); - RMApp app0 = rm0.submitApp(200); + RMApp app0 = MockRMAppSubmitter.submitWithMemory(200, rm0); rm0.waitForState(app0.getApplicationId(), RMAppState.ACCEPTED); RMAppAttempt attempt0 = app0.getCurrentAppAttempt(); @@ -262,10 +262,21 @@ private void amRestartTests(boolean keepRunningContainers) MockRM rm = new MockRM(conf); rm.start(); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm) + .withName("name") + .withUser("user") + .withAcls(new HashMap()) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(-1) + .withCredentials(null) + .withAppType("MAPREDUCE") + .withWaitForAccepted(false) + .withKeepContainers(keepRunningContainers) + .build(); RMApp app = - rm.submitApp(200, "name", "user", - new HashMap(), false, "default", -1, - null, "MAPREDUCE", false, keepRunningContainers); + MockRMAppSubmitter.submit(rm, data); MockNM nm = new MockNM("127.0.0.1:1234", 10240, rm.getResourceTrackerService()); nm.registerNode(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestDecommissioningNodesWatcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestDecommissioningNodesWatcher.java index 0695689e165..ca94938ce93 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestDecommissioningNodesWatcher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestDecommissioningNodesWatcher.java @@ -65,7 +65,7 @@ public void testDecommissioningNodesWatcher() throws Exception { rm.waitForState(id1, NodeState.RUNNING); - RMApp app = rm.submitApp(2000); + RMApp app = MockRMAppSubmitter.submitWithMemory(2000, rm); MockAM am = MockRM.launchAndRegisterAM(app, rm, nm1); NodeStatus nodeStatus = createNodeStatus(id1, app, 3); @@ -125,7 +125,8 @@ public void testDecommissioningNodesWatcherWithPreviousRunningApps() rm.waitForState(id1, NodeState.RUNNING); - RMApp app = rm.submitApp(2000); + RMApp app = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(2000, rm).build()); MockAM am = MockRM.launchAndRegisterAM(app, rm, nm1); NodeStatus nodeStatus = createNodeStatus(id1, app, 3); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestKillApplicationWithRMHA.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestKillApplicationWithRMHA.java index 979d8c7b974..5a3d5c59d79 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestKillApplicationWithRMHA.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestKillApplicationWithRMHA.java @@ -22,6 +22,7 @@ import static org.junit.Assert.fail; import java.io.IOException; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; @@ -62,12 +63,23 @@ public void testKillAppWhenFailoverHappensAtNewState() nm1.registerNode(); // Submit the application + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("") + .withUser(UserGroupInformation + .getCurrentUser().getShortUserName()) + .withAcls(null) + .withUnmanaged(false) + .withQueue(null) + .withMaxAppAttempts(configuration.getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, + YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS)) + .withCredentials(null) + .withAppType(null) + .withWaitForAccepted(false) + .withKeepContainers(false) + .build(); RMApp app0 = - rm1.submitApp(200, "", UserGroupInformation - .getCurrentUser().getShortUserName(), null, false, null, - configuration.getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, - false, false); + MockRMAppSubmitter.submit(rm1, data); // failover and kill application // When FailOver happens, the state of this application is NEW, @@ -93,7 +105,7 @@ public void testKillAppWhenFailoverHappensAtRunningState() nm1.registerNode(); // create app and launch the AM - RMApp app0 = rm1.submitApp(200); + RMApp app0 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am0 = launchAM(app0, rm1, nm1); // failover and kill application @@ -116,7 +128,7 @@ public void testKillAppWhenFailoverHappensAtFinalState() nm1.registerNode(); // create app and launch the AM - RMApp app0 = rm1.submitApp(200); + RMApp app0 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am0 = launchAM(app0, rm1, nm1); // kill the app. @@ -147,7 +159,7 @@ public void testKillAppWhenFailOverHappensDuringApplicationKill() nm1.registerNode(); // create app and launch the AM - RMApp app0 = rm1.submitApp(200); + RMApp app0 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am0 = launchAM(app0, rm1, nm1); // ensure that the app is in running state diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestLeaderElectorService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestLeaderElectorService.java index 4cd1df607fb..4912a444095 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestLeaderElectorService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestLeaderElectorService.java @@ -126,7 +126,14 @@ public synchronized void storeApplicationStateInternal(ApplicationId rm2 = startRM("rm2", HAServiceState.STANDBY); // submit an app which will trigger state-store failure. - rm1.submitApp(200, "app1", "user1", null, "default", false); + MockRMAppSubmitter.submit(rm1, + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("app1") + .withUser("user1") + .withAcls(null) + .withQueue("default") + .withWaitForAccepted(false) + .build()); waitFor(rm1, HAServiceState.STANDBY); // rm2 should become active; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestMoveApplication.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestMoveApplication.java index 05b25df3a33..a078fb2ed7b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestMoveApplication.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestMoveApplication.java @@ -124,7 +124,7 @@ public void testMoveTooLate() throws Exception { void testMoveSuccessful() throws Exception { MockRM rm1 = new MockRM(conf); rm1.start(); - RMApp app = rm1.submitApp(1024); + RMApp app = MockRMAppSubmitter.submitWithMemory(1024, rm1); ClientRMService clientRMService = rm1.getClientRMService(); // FIFO scheduler does not support moves clientRMService diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestNodeBlacklistingOnAMFailures.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestNodeBlacklistingOnAMFailures.java index e7d666ad8ad..1ac11baf98a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestNodeBlacklistingOnAMFailures.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestNodeBlacklistingOnAMFailures.java @@ -58,6 +58,33 @@ */ public class TestNodeBlacklistingOnAMFailures { + private RMApp submitAppWithAMResourceRequests(MockRM rm, List amResourceRequests) throws Exception { + int maxAttempts = rm.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS); + MockRMAppSubmissionData data = MockRMAppSubmissionData.Builder.create() + .withAmResourceRequests(amResourceRequests) + .withName("app1") + .withUser("user") + .withAcls(null) + .withUnmanaged(false) + .withQueue(null) + .withMaxAppAttempts(maxAttempts) + .withCredentials(null) + .withAppType(null) + .withWaitForAccepted(true) + .withKeepContainers(false) + .withIsAppIdProvided(false) + .withApplicationId(null) + .withAttemptFailuresValidityInterval(0) + .withLogAggregationContext(null) + .withCancelTokensWhenComplete(true) + .withPriority(amResourceRequests.get(0).getPriority()) + .withAmLabel(amResourceRequests.get(0).getNodeLabelExpression()) + .withApplicationTimeouts(null) + .withTokensConf(null) + .build(); + return MockRMAppSubmitter.submit(rm, data); + } + @Before public void setup() { QueueMetrics.clearQueueMetrics(); @@ -87,7 +114,7 @@ public void testNodeBlacklistingOnAMFailure() throws Exception { new MockNM("127.0.0.2:2345", 8000, rm.getResourceTrackerService()); nm2.registerNode(); - RMApp app = rm.submitApp(200); + RMApp app = MockRMAppSubmitter.submitWithMemory(200, rm); MockAM am1 = MockRM.launchAndRegisterAM(app, rm, nm1); ContainerId amContainerId = @@ -215,7 +242,7 @@ public void testNodeBlacklistingOnAMFailureStrictNodeLocality() reqs.add(anyReq); reqs.add(rackReq); reqs.add(nodeReq); - RMApp app = rm.submitApp(reqs); + RMApp app = submitAppWithAMResourceRequests(rm, reqs); MockAM am1 = MockRM.launchAndRegisterAM(app, rm, nm2); ContainerId amContainerId = @@ -302,7 +329,7 @@ public void testNodeBlacklistingOnAMFailureRelaxedNodeLocality() reqs.add(anyReq); reqs.add(rackReq); reqs.add(nodeReq); - RMApp app = rm.submitApp(reqs); + RMApp app = submitAppWithAMResourceRequests(rm, reqs); MockAM am1 = MockRM.launchAndRegisterAM(app, rm, nm2); ContainerId amContainerId = @@ -363,7 +390,7 @@ public void testNoBlacklistingForNonSystemErrors() throws Exception { new MockNM("127.0.0.1:1234", 8000, rm.getResourceTrackerService()); node.registerNode(); - RMApp app = rm.submitApp(200); + RMApp app = MockRMAppSubmitter.submitWithMemory(200, rm); ApplicationId appId = app.getApplicationId(); int numAppAttempts = 1; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestOpportunisticContainerAllocatorAMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestOpportunisticContainerAllocatorAMService.java index 70782440423..73ab2ec3507 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestOpportunisticContainerAllocatorAMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestOpportunisticContainerAllocatorAMService.java @@ -191,8 +191,15 @@ public void testContainerPromoteAndDemoteBeforeContainerStart() throws Exception OpportunisticContainerAllocatorAMService amservice = (OpportunisticContainerAllocatorAMService) rm .getApplicationMasterService(); - RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default"); - + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm2); ResourceScheduler scheduler = rm.getResourceScheduler(); @@ -343,8 +350,15 @@ public void testContainerPromoteAfterContainerStart() throws Exception { OpportunisticContainerAllocatorAMService amservice = (OpportunisticContainerAllocatorAMService) rm .getApplicationMasterService(); - RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default"); - + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm2); ResourceScheduler scheduler = rm.getResourceScheduler(); @@ -453,8 +467,15 @@ public void testContainerPromoteAfterContainerComplete() throws Exception { OpportunisticContainerAllocatorAMService amservice = (OpportunisticContainerAllocatorAMService) rm .getApplicationMasterService(); - RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default"); - + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm2); ResourceScheduler scheduler = rm.getResourceScheduler(); @@ -550,7 +571,15 @@ public void testContainerAutoUpdateContainer() throws Exception { OpportunisticContainerAllocatorAMService amservice = (OpportunisticContainerAllocatorAMService) rm .getApplicationMasterService(); - RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); ResourceScheduler scheduler = rm.getResourceScheduler(); RMNode rmNode1 = rm.getRMContext().getRMNodes().get(nm1.getNodeId()); @@ -717,7 +746,13 @@ public void testOpportunisticSchedulerMetrics() throws Exception { OpportunisticContainerAllocatorAMService amservice = (OpportunisticContainerAllocatorAMService) rm .getApplicationMasterService(); - RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default"); + RMApp app1 = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .build()); ApplicationAttemptId attemptId = app1.getCurrentAppAttempt().getAppAttemptId(); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm2); @@ -787,7 +822,13 @@ public void testAMCrashDuringAllocate() throws Exception { MockNM nm = new MockNM("h:1234", 4096, rm.getResourceTrackerService()); nm.registerNode(); - RMApp app = rm.submitApp(1 * GB, "app", "user", null, "default"); + RMApp app = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .build()); ApplicationAttemptId attemptId0 = app.getCurrentAppAttempt().getAppAttemptId(); MockAM am = MockRM.launchAndRegisterAM(app, rm, nm); @@ -824,7 +865,15 @@ public void testNodeRemovalDuringAllocate() throws Exception { OpportunisticContainerAllocatorAMService amservice = (OpportunisticContainerAllocatorAMService) rm .getApplicationMasterService(); - RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); ApplicationAttemptId attemptId = app1.getCurrentAppAttempt().getAppAttemptId(); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm2); @@ -877,7 +926,13 @@ public void testAppAttemptRemovalAfterNodeRemoval() throws Exception { nm.registerNode(); nm.nodeHeartbeat(oppContainersStatus, true); - RMApp app = rm.submitApp(1 * GB, "app", "user", null, "default"); + RMApp app = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .build()); ApplicationAttemptId attemptId = app.getCurrentAppAttempt().getAppAttemptId(); MockAM am = MockRM.launchAndRegisterAM(app, rm, nm); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java index 49144a3c415..2078d327756 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java @@ -123,8 +123,8 @@ public void testAppWithNoContainers() throws Exception { MockRM rm = new MockRM(conf); rm.start(); MockNM nm1 = rm.registerNode("h1:1234", 5120); - - RMApp app = rm.submitApp(2000); + + RMApp app = MockRMAppSubmitter.submitWithMemory(2000, rm); //kick the scheduling nm1.nodeHeartbeat(true); @@ -146,8 +146,8 @@ public void testAppOnMultiNode() throws Exception { rm.start(); MockNM nm1 = rm.registerNode("h1:1234", 5120); MockNM nm2 = rm.registerNode("h2:5678", 10240); - - RMApp app = rm.submitApp(2000); + + RMApp app = MockRMAppSubmitter.submitWithMemory(2000, rm); //kick the scheduling nm1.nodeHeartbeat(true); @@ -203,7 +203,7 @@ public void testNMTokenSentForNormalContainer() throws Exception { MockRM rm = new MockRM(conf); rm.start(); MockNM nm1 = rm.registerNode("h1:1234", 5120); - RMApp app = rm.submitApp(2000); + RMApp app = MockRMAppSubmitter.submitWithMemory(2000, rm); RMAppAttempt attempt = app.getCurrentAppAttempt(); // Call getNewContainerId to increase container Id so that the AM container @@ -259,7 +259,7 @@ public void testNMToken() throws Exception { rm.getRMContext().getNMTokenSecretManager(); // submitting new application - RMApp app = rm.submitApp(1000); + RMApp app = MockRMAppSubmitter.submitWithMemory(1000, rm); // start scheduling. nm1.nodeHeartbeat(true); @@ -440,10 +440,10 @@ public void testActivatingApplicationAfterAddingNM() throws Exception { rm1.start(); // app that gets launched - RMApp app1 = rm1.submitApp(200); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(200, rm1); // app that does not get launched - RMApp app2 = rm1.submitApp(200); + RMApp app2 = MockRMAppSubmitter.submitWithMemory(200, rm1); // app1 and app2 should be scheduled, but because no resource is available, // they are not activated. @@ -484,7 +484,7 @@ public void testInvalidateAMHostPortWhenAMFailedOrKilled() throws Exception { rm1.start(); // a succeeded app - RMApp app1 = rm1.submitApp(200); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService()); nm1.registerNode(); @@ -492,14 +492,14 @@ public void testInvalidateAMHostPortWhenAMFailedOrKilled() throws Exception { MockRM.finishAMAndVerifyAppState(app1, rm1, nm1, am1); // a failed app - RMApp app2 = rm1.submitApp(200); + RMApp app2 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1); nm1.nodeHeartbeat(am2.getApplicationAttemptId(), 1, ContainerState.COMPLETE); rm1.waitForState(am2.getApplicationAttemptId(), RMAppAttemptState.FAILED); rm1.waitForState(app2.getApplicationId(), RMAppState.FAILED); // a killed app - RMApp app3 = rm1.submitApp(200); + RMApp app3 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am3 = MockRM.launchAndRegisterAM(app3, rm1, nm1); rm1.killApp(app3.getApplicationId()); rm1.waitForState(app3.getApplicationId(), RMAppState.KILLED); @@ -538,7 +538,7 @@ public void testInvalidatedAMHostPortOnAMRestart() throws Exception { nm1.registerNode(); // a failed app - RMApp app2 = rm1.submitApp(200); + RMApp app2 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1); nm1 .nodeHeartbeat(am2.getApplicationAttemptId(), 1, ContainerState.COMPLETE); @@ -605,7 +605,7 @@ protected Dispatcher createDispatcher() { nm1.registerNode(); // a failed app - RMApp application = rm.submitApp(200); + RMApp application = MockRMAppSubmitter.submitWithMemory(200, rm); MockAM am = MockRM.launchAM(application, rm, nm1); rm.waitForState(am.getApplicationAttemptId(), RMAppAttemptState.LAUNCHED); nm1.nodeHeartbeat(am.getApplicationAttemptId(), 1, ContainerState.RUNNING); @@ -683,7 +683,7 @@ protected Dispatcher createDispatcher() { MockNM nm1 = new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService()); nm1.registerNode(); - RMApp app1 = rm1.submitApp(200); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); rm1.killApp(app1.getApplicationId()); @@ -737,7 +737,7 @@ protected Dispatcher createDispatcher() { MockNM nm1 = new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService()); nm1.registerNode(); - RMApp app1 = rm1.submitApp(200); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); rm1.killApp(app1.getApplicationId()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java index 4b11296efbd..b44a8aed815 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java @@ -142,7 +142,7 @@ private void checkActiveRMFunctionality() throws Exception { try { rm.getNewAppId(); nm = rm.registerNode("127.0.0.1:1", 2048); - app = rm.submitApp(1024); + app = MockRMAppSubmitter.submitWithMemory(1024, rm); attempt = app.getCurrentAppAttempt(); rm.waitForState(attempt.getAppAttemptId(), RMAppAttemptState.SCHEDULED); } catch (Exception e) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHAForAsyncScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHAForAsyncScheduler.java index 36f17623281..d46b8266c2e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHAForAsyncScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHAForAsyncScheduler.java @@ -136,11 +136,21 @@ public void testAsyncScheduleThreadStateAfterRMHATransit() throws Exception { } private RMApp submitAppAndCheckLaunched(MockRM rm) throws Exception { - RMApp app = rm.submitApp(200, "", - UserGroupInformation.getCurrentUser().getShortUserName(), null, false, - "default", configuration.getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, false, - false); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm) + .withName("") + .withUser(UserGroupInformation.getCurrentUser().getShortUserName()) + .withAcls(null) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(configuration.getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, + YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS)) + .withCredentials(null) + .withAppType(null) + .withWaitForAccepted(false) + .withKeepContainers(false) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); rm.waitForState(app.getApplicationId(), RMAppState.ACCEPTED); RMAppAttempt attempt = app.getCurrentAppAttempt(); rm.sendAMLaunched(attempt.getAppAttemptId()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHATimelineCollectors.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHATimelineCollectors.java index a0cd0486d3a..d4f156bd0b1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHATimelineCollectors.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHATimelineCollectors.java @@ -60,14 +60,14 @@ public void testRebuildCollectorDataOnFailover() throws Exception { = new MockNM("127.0.0.1:1234", 15120, rm2.getResourceTrackerService()); MockNM nm2 = new MockNM("127.0.0.1:5678", 15121, rm2.getResourceTrackerService()); - RMApp app1 = rm1.submitApp(1024); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(1024, rm1); String collectorAddr1 = "1.2.3.4:5"; AppCollectorData data1 = AppCollectorData.newInstance( app1.getApplicationId(), collectorAddr1); nm1.addRegisteringCollector(app1.getApplicationId(), data1); String collectorAddr2 = "5.4.3.2:1"; - RMApp app2 = rm1.submitApp(1024); + RMApp app2 = MockRMAppSubmitter.submitWithMemory(1024, rm1); AppCollectorData data2 = AppCollectorData.newInstance( app2.getApplicationId(), collectorAddr2, rm1.getStartTime(), 1); nm1.addRegisteringCollector(app2.getApplicationId(), data2); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java index 6127214fab0..7617b495792 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java @@ -194,6 +194,36 @@ private MockRM createMockRM(YarnConfiguration config) { return rm; } + private RMApp submitApp(MockRM rm, List amResourceRequests, + String appNodeLabel) throws Exception { + int maxAttempts = rm.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS); + MockRMAppSubmissionData data = MockRMAppSubmissionData.Builder.create() + .withAmResourceRequests(amResourceRequests) + .withName("app1") + .withUser("user") + .withAcls(null) + .withUnmanaged(false) + .withQueue(null) + .withMaxAppAttempts(maxAttempts) + .withCredentials(null) + .withAppType(null) + .withWaitForAccepted(true) + .withKeepContainers(false) + .withIsAppIdProvided(false) + .withApplicationId(null) + .withAttemptFailuresValidityInterval(0) + .withLogAggregationContext(null) + .withCancelTokensWhenComplete(true) + .withPriority(amResourceRequests.get(0).getPriority()) + .withAmLabel(amResourceRequests.get(0).getNodeLabelExpression()) + .withApplicationTimeouts(null) + .withTokensConf(null) + .withApplicationTags(null) + .withAppNodeLabel(appNodeLabel) + .build(); + return MockRMAppSubmitter.submit(rm, data); + } + @Test (timeout=180000) public void testRMRestart() throws Exception { conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, @@ -217,7 +247,7 @@ public void testRMRestart() throws Exception { nm2.registerNode(); // nm2 will not heartbeat with RM1 // create app that will finish and the final state should be saved. - RMApp app0 = rm1.submitApp(200); + RMApp app0 = MockRMAppSubmitter.submitWithMemory(200, rm1); RMAppAttempt attempt0 = app0.getCurrentAppAttempt(); // spot check that app is saved Assert.assertEquals(1, rmAppState.size()); @@ -227,7 +257,7 @@ public void testRMRestart() throws Exception { finishApplicationMaster(app0, rm1, nm1, am0); // create app that gets launched and does allocate before RM restart - RMApp app1 = rm1.submitApp(200); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(200, rm1); // assert app1 info is saved ApplicationStateData appState = rmAppState.get(app1.getApplicationId()); Assert.assertNotNull(appState); @@ -268,7 +298,7 @@ public void testRMRestart() throws Exception { } // create app that does not get launched by RM before RM restart - RMApp app2 = rm1.submitApp(200); + RMApp app2 = MockRMAppSubmitter.submitWithMemory(200, rm1); // assert app2 info is saved appState = rmAppState.get(app2.getApplicationId()); @@ -279,9 +309,17 @@ public void testRMRestart() throws Exception { .getApplicationId()); // create unmanaged app - RMApp appUnmanaged = rm1.submitApp(200, "someApp", "someUser", null, true, - null, conf.getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null); + RMApp appUnmanaged = MockRMAppSubmitter.submit(rm1, + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("someApp") + .withUser("someUser") + .withAcls(null) + .withUnmanaged(true) + .withQueue(null) + .withMaxAppAttempts(conf.getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, + YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS)) + .withCredentials(null) + .build()); ApplicationAttemptId unmanagedAttemptId = appUnmanaged.getCurrentAppAttempt().getAppAttemptId(); // assert appUnmanaged info is saved @@ -483,7 +521,7 @@ public void testAppReportNodeLabelRMRestart() throws Exception { Resource.newInstance(200, 1), 1, true, amLabel.getName()); ArrayList resReqs = new ArrayList<>(); resReqs.add(amResourceRequest); - RMApp app0 = rm1.submitApp(resReqs, appLabel.getName()); + RMApp app0 = submitApp(rm1, resReqs, appLabel.getName()); rm1.killApp(app0.getApplicationId()); rm1.waitForState(app0.getApplicationId(), RMAppState.KILLED); // start new RM @@ -509,7 +547,14 @@ public void testUnManagedRMRestart() throws Exception { rm1.start(); // create app and launch the AM RMApp app0 = - rm1.submitApp(null, "name", "user", new HashMap<>(), true, "default"); + MockRMAppSubmitter.submit(rm1, + MockRMAppSubmissionData.Builder.createWithDefaults(null, rm1) + .withName("name") + .withUser("user") + .withAcls(new HashMap<>()) + .withUnmanaged(true) + .withQueue("default") + .build()); rm1.killApp(app0.getApplicationId()); rm1.waitForState(app0.getApplicationId(), RMAppState.KILLED); // start new RM @@ -540,10 +585,21 @@ public void testRMRestartAppRunningAMFailed() throws Exception { nm1.registerNode(); // create app and launch the AM + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("name") + .withUser("user") + .withAcls(new HashMap()) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(-1) + .withCredentials(null) + .withAppType("MAPREDUCE") + .withWaitForAccepted(true) + .withKeepContainers(true) + .build(); RMApp app0 = - rm1.submitApp(200, "name", "user", - new HashMap(), false, "default", -1, - null, "MAPREDUCE", true, true); + MockRMAppSubmitter.submit(rm1, data); MockAM am0 = launchAM(app0, rm1, nm1); // fail the AM by sending CONTAINER_FINISHED event without registering. @@ -598,7 +654,7 @@ public void testRMRestartWaitForPreviousAMToFinish() throws Exception { nm1.registerNode(); // submitting app - RMApp app1 = rm1.submitApp(200); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(200, rm1); rm1.waitForState(app1.getApplicationId(), RMAppState.ACCEPTED); MockAM am1 = launchAM(app1, rm1, nm1); nm1.nodeHeartbeat(am1.getApplicationAttemptId(), 1, ContainerState.COMPLETE); @@ -706,7 +762,7 @@ public Boolean get() { // It will be saved only when we launch AM. // submitting app but not starting AM for it. - RMApp app2 = rm3.submitApp(200); + RMApp app2 = MockRMAppSubmitter.submitWithMemory(200, rm3); rm3.waitForState(app2.getApplicationId(), RMAppState.ACCEPTED); Assert.assertEquals(1, app2.getAppAttempts().size()); Assert.assertEquals(0, @@ -773,7 +829,7 @@ public void updateApplicationStateInternal(ApplicationId appId, MockRM rm1 = createMockRM(conf, memStore); rm1.start(); MockNM nm1 = rm1.registerNode("127.0.0.1:1234", 15120); - RMApp app0 = rm1.submitApp(200); + RMApp app0 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am0 = MockRM.launchAndRegisterAM(app0, rm1, nm1); FinishApplicationMasterRequest req = FinishApplicationMasterRequest.newInstance( @@ -814,7 +870,7 @@ public void testRMRestartFailedApp() throws Exception { nm1.registerNode(); // create app and launch the AM - RMApp app0 = rm1.submitApp(200); + RMApp app0 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am0 = launchAM(app0, rm1, nm1); // fail the AM by sending CONTAINER_FINISHED event without registering. @@ -862,7 +918,7 @@ public void testRMRestartKilledApp() throws Exception{ nm1.registerNode(); // create app and launch the AM - RMApp app0 = rm1.submitApp(200); + RMApp app0 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am0 = launchAM(app0, rm1, nm1); // kill the app. @@ -918,9 +974,18 @@ public synchronized void updateApplicationAttemptStateInternal( rm1.start(); // create app RMApp app0 = - rm1.submitApp(200, "name", "user", - new HashMap(), false, "default", -1, - null, "MAPREDUCE", false); + MockRMAppSubmitter.submit(rm1, + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("name") + .withUser("user") + .withAcls(new HashMap()) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(-1) + .withCredentials(null) + .withAppType("MAPREDUCE") + .withWaitForAccepted(false) + .build()); // kill the app. rm1.killApp(app0.getApplicationId()); rm1.waitForState(app0.getApplicationId(), RMAppState.KILLED); @@ -952,7 +1017,7 @@ public void testRMRestartSucceededApp() throws Exception { nm1.registerNode(); // create an app and finish the app. - RMApp app0 = rm1.submitApp(200); + RMApp app0 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am0 = launchAM(app0, rm1, nm1); // unregister am @@ -1003,14 +1068,32 @@ protected SystemMetricsPublisher createSystemMetricsPublisher() { (MockMemoryRMStateStore) rm1.getRMStateStore(); // a succeeded app. - RMApp app0 = rm1.submitApp(200, "name", "user", null, - false, "default", 1, null, "myType"); + RMApp app0 = MockRMAppSubmitter.submit(rm1, + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("name") + .withUser("user") + .withAcls(null) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(1) + .withCredentials(null) + .withAppType("myType") + .build()); MockAM am0 = launchAM(app0, rm1, nm1); finishApplicationMaster(app0, rm1, nm1, am0); // a failed app. - RMApp app1 = rm1.submitApp(200, "name", "user", null, - false, "default", 1, null, "myType"); + RMApp app1 = MockRMAppSubmitter.submit(rm1, + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("name") + .withUser("user") + .withAcls(null) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(1) + .withCredentials(null) + .withAppType("myType") + .build()); MockAM am1 = launchAM(app1, rm1, nm1); // fail the AM by sending CONTAINER_FINISHED event without registering. nm1.nodeHeartbeat(am1.getApplicationAttemptId(), 1, ContainerState.COMPLETE); @@ -1018,8 +1101,17 @@ protected SystemMetricsPublisher createSystemMetricsPublisher() { rm1.waitForState(app1.getApplicationId(), RMAppState.FAILED); // a killed app. - RMApp app2 = rm1.submitApp(200, "name", "user", null, - false, "default", 1, null, "myType"); + RMApp app2 = MockRMAppSubmitter.submit(rm1, + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("name") + .withUser("user") + .withAcls(null) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(1) + .withCredentials(null) + .withAppType("myType") + .build()); MockAM am2 = launchAM(app2, rm1, nm1); rm1.killApp(app2.getApplicationId()); rm1.waitForState(app2.getApplicationId(), RMAppState.KILLED); @@ -1162,13 +1254,27 @@ public void testRMRestartOnMaxAppAttempts() throws Exception { nm1.registerNode(); // submit an app with maxAppAttempts equals to 1 - RMApp app1 = rm1.submitApp(200, "name", "user", - new HashMap(), false, "default", 1, - null); + RMApp app1 = MockRMAppSubmitter.submit(rm1, + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("name") + .withUser("user") + .withAcls(new HashMap()) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(1) + .withCredentials(null) + .build()); // submit an app with maxAppAttempts equals to -1 - RMApp app2 = rm1.submitApp(200, "name", "user", - new HashMap(), false, "default", -1, - null); + RMApp app2 = MockRMAppSubmitter.submit(rm1, + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("name") + .withUser("user") + .withAcls(new HashMap()) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(-1) + .withCredentials(null) + .build()); // assert app1 info is saved ApplicationStateData appState = rmAppState.get(app1.getApplicationId()); @@ -1232,9 +1338,16 @@ public void testRMRestartTimelineCollectorContext() throws Exception { nm1.registerNode(); // submit an app. - RMApp app = rm1.submitApp(200, "name", "user", - new HashMap(), false, "default", -1, - null); + RMApp app = MockRMAppSubmitter.submit(rm1, + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("name") + .withUser("user") + .withAcls(new HashMap()) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(-1) + .withCredentials(null) + .build()); // Check if app info has been saved. ApplicationStateData appState = rmAppState.get(app.getApplicationId()); Assert.assertNotNull(appState); @@ -1321,8 +1434,16 @@ public void testDelegationTokenRestoredInDelegationTokenRenewer() tokenSet.add(token2); // submit an app with customized credential - RMApp app = rm1.submitApp(200, "name", "user", - new HashMap(), false, "default", 1, ts); + RMApp app = MockRMAppSubmitter.submit(rm1, + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("name") + .withUser("user") + .withAcls(new HashMap()) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(1) + .withCredentials(ts) + .build()); // assert app info is saved ApplicationStateData appState = rmAppState.get(app.getApplicationId()); @@ -1389,9 +1510,16 @@ public void testAppAttemptTokensRestoredOnRMRestart() throws Exception { nm1.registerNode(); // submit an app + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("name") + .withUser("user") + .withAcls(new HashMap()) + .withQueue("default") + .withUnmanaged(false) + .build(); RMApp app1 = - rm1.submitApp(200, "name", "user", - new HashMap(), "default"); + MockRMAppSubmitter.submit(rm1, data); // assert app info is saved ApplicationStateData appState = rmAppState.get(app1.getApplicationId()); @@ -1492,8 +1620,16 @@ public void testRMDelegationTokenRestoredOnRMRestart() throws Exception { tokenIdentSet.add(dtId1); // submit an app with customized credential - RMApp app = rm1.submitApp(200, "name", "user", - new HashMap(), false, "default", 1, ts); + RMApp app = MockRMAppSubmitter.submit(rm1, + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("name") + .withUser("user") + .withAcls(new HashMap()) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(1) + .withCredentials(ts) + .build()); // assert app info is saved ApplicationStateData appState = rmAppState.get(app.getApplicationId()); @@ -1624,8 +1760,16 @@ public void testAppSubmissionWithOldDelegationTokenAfterRMRestart() // submit an app with the old delegation token got from previous RM. Credentials ts = new Credentials(); ts.addToken(token1.getService(), token1); - RMApp app = rm2.submitApp(200, "name", "user", - new HashMap(), false, "default", 1, ts); + RMApp app = MockRMAppSubmitter.submit(rm2, + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm2) + .withName("name") + .withUser("user") + .withAcls(new HashMap()) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(1) + .withCredentials(ts) + .build()); rm2.waitForState(app.getApplicationId(), RMAppState.ACCEPTED); } @@ -1665,9 +1809,18 @@ protected void handleStoreEvent(RMStateStoreEvent event) { final int NUM_APPS = 5; for (int i = 0; i < NUM_APPS; i++) { - RMApp app = rm1.submitApp(200, "name", "user", - new HashMap(), false, - "default", -1, null, "MAPREDUCE", false); + RMApp app = MockRMAppSubmitter.submit(rm1, + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("name") + .withUser("user") + .withAcls(new HashMap()) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(-1) + .withCredentials(null) + .withAppType("MAPREDUCE") + .withWaitForAccepted(false) + .build()); appList.add(app); rm1.waitForState(app.getApplicationId(), RMAppState.NEW_SAVING); } @@ -1710,7 +1863,7 @@ public void testFinishedAppRemovalAfterRMRestart() throws Exception { nm1.registerNode(); // create an app and finish the app. - RMApp app0 = rm1.submitApp(200); + RMApp app0 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am0 = launchAM(app0, rm1, nm1); finishApplicationMaster(app0, rm1, nm1, am0); @@ -1728,7 +1881,7 @@ public void testFinishedAppRemovalAfterRMRestart() throws Exception { rm2.waitForState(app0.getApplicationId(), RMAppState.FINISHED); // create one more app and finish the app. - RMApp app1 = rm2.submitApp(200); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(200, rm2); MockAM am1 = launchAM(app1, rm2, nm1); finishApplicationMaster(app1, rm2, nm1, am1); rm2.drainEvents(); @@ -1778,8 +1931,17 @@ public void testClientRetryOnKillingApplication() throws Exception { nm1.registerNode(); RMApp app1 = - rm1.submitApp(200, "name", "user", null, false, "default", 1, null, - "myType"); + MockRMAppSubmitter.submit(rm1, + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("name") + .withUser("user") + .withAcls(null) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(1) + .withCredentials(null) + .withAppType("myType") + .build()); MockAM am1 = launchAM(app1, rm1, nm1); KillApplicationResponse response; @@ -1828,9 +1990,18 @@ protected DelegationTokenRenewer createDelegationTokenRenewer() { (MockMemoryRMStateStore) rm1.getRMStateStore(); RMApp app1 = null; try { - app1 = rm1.submitApp(200, "name", "user", - new HashMap(), false, "default", -1, - null, "MAPREDUCE", false); + app1 = MockRMAppSubmitter.submit(rm1, + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("name") + .withUser("user") + .withAcls(new HashMap()) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(-1) + .withCredentials(null) + .withAppType("MAPREDUCE") + .withWaitForAccepted(false) + .build()); Assert.fail(); } catch (Exception e) { @@ -1915,7 +2086,7 @@ public void testQueueMetricsOnRMRestart() throws Exception { assertQueueMetrics(qm1, 0, 0, 0, 0); // create app that gets launched and does allocate before RM restart - RMApp app1 = rm1.submitApp(200); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(200, rm1); // Need to wait first for AppAttempt to be started (RMAppState.ACCEPTED) // and then for it to reach RMAppAttemptState.SCHEDULED // inorder to ensure appsPending metric is incremented @@ -2110,7 +2281,7 @@ public void testSynchronouslyRenewDTOnRecovery() throws Exception { final MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService()); nm1.registerNode(); - RMApp app0 = rm1.submitApp(200); + RMApp app0 = MockRMAppSubmitter.submitWithMemory(200, rm1); final MockAM am0 = MockRM.launchAndRegisterAM(app0, rm1, nm1); MockRM rm2 = new TestSecurityMockRM(conf, rm1.getRMStateStore()) { @@ -2367,7 +2538,7 @@ public void testRMRestartFailAppAttempt() throws Exception { nm1.registerNode(); // create app and launch the AM - RMApp app0 = rm1.submitApp(200); + RMApp app0 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am0 = launchAM(app0, rm1, nm1); ApplicationId applicationId = app0.getApplicationId(); @@ -2496,7 +2667,7 @@ public void testRMRestartAfterPreemption() throws Exception { nm1.registerNode(); int CONTAINER_MEMORY = 1024; // create app and launch the AM - RMApp app0 = rm1.submitApp(CONTAINER_MEMORY); + RMApp app0 = MockRMAppSubmitter.submitWithMemory(CONTAINER_MEMORY, rm1); MockAM am0 = MockRM.launchAM(app0, rm1, nm1); nm1.nodeHeartbeat(am0.getApplicationAttemptId(), 1, ContainerState.COMPLETE); @@ -2550,7 +2721,7 @@ public void testRMRestartOnMissingAttempts() throws Exception { nm1.registerNode(); // create an app and finish the app. - RMApp app0 = rm1.submitApp(200); + RMApp app0 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am0 = launchAndFailAM(app0, rm1, nm1); MockAM am1 = launchAndFailAM(app0, rm1, nm1); @@ -2642,7 +2813,15 @@ protected RMNodeLabelsManager createNodeLabelManager() { MockNM nm1 = rm1.registerNode("h1:1234", 8000); // label = x // submit an application with specifying am node label expression as "x" - RMApp app1 = rm1.submitApp(200, "someApp", "someUser", null, "a1", "x"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("someApp") + .withUser("someUser") + .withAcls(null) + .withQueue("a1") + .withAmLabel("x") + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); // check am container allocated with correct node label expression MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); ContainerId amContainerId1 = @@ -2708,8 +2887,13 @@ public void testRMRestartAfterPriorityChangesInAllocatedResponse() // Submit an application Priority appPriority1 = Priority.newInstance(5); - RMApp app1 = rm.submitApp(2048, appPriority1, - getCreds(), getTokensConf()); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(2048, rm) + .withPriority(appPriority1) + .withCredentials(getCreds()) + .withTokensConf(getTokensConf()) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); nm1.nodeHeartbeat(true); RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); @@ -2781,7 +2965,7 @@ public void testRMRestartAfterUpdateTrackingUrl() throws Exception { // Register node1 MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * 1024); - RMApp app1 = rm.submitApp(2048); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(2048, rm); nm1.nodeHeartbeat(true); RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java index 1fd34a0bb08..f775746a833 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java @@ -424,7 +424,7 @@ public void testGracefulDecommissionWithApp() throws Exception { rm.waitForState(id3, NodeState.RUNNING); // Create an app and launch two containers on host1. - RMApp app = rm.submitApp(2000); + RMApp app = MockRMAppSubmitter.submitWithMemory(2000, rm); MockAM am = MockRM.launchAndRegisterAM(app, rm, nm1); ApplicationAttemptId aaid = app.getCurrentAppAttempt().getAppAttemptId(); nm1.nodeHeartbeat(aaid, 2, ContainerState.RUNNING); @@ -1622,13 +1622,13 @@ public void testNodeHeartbeatForAppCollectorsMap() throws Exception { RMNodeImpl node2 = (RMNodeImpl) rm.getRMContext().getRMNodes().get(nm2.getNodeId()); - RMAppImpl app1 = (RMAppImpl) rm.submitApp(1024); + RMAppImpl app1 = (RMAppImpl) MockRMAppSubmitter.submitWithMemory(1024, rm); String collectorAddr1 = "1.2.3.4:5"; app1.setCollectorData(AppCollectorData.newInstance( app1.getApplicationId(), collectorAddr1)); String collectorAddr2 = "5.4.3.2:1"; - RMAppImpl app2 = (RMAppImpl) rm.submitApp(1024); + RMAppImpl app2 = (RMAppImpl) MockRMAppSubmitter.submitWithMemory(1024, rm); app2.setCollectorData(AppCollectorData.newInstance( app2.getApplicationId(), collectorAddr2)); @@ -1752,7 +1752,11 @@ public void testHandleContainerStatusInvalidCompletions() throws Exception { spy(rm.getRMContext().getDispatcher().getEventHandler()); // Case 1: Unmanaged AM - RMApp app = rm.submitApp(1024, true); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm) + .withUnmanaged(true) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); // Case 1.1: AppAttemptId is null NMContainerStatus report = @@ -1776,7 +1780,7 @@ public void testHandleContainerStatusInvalidCompletions() throws Exception { verify(handler, never()).handle((Event)any()); // Case 2: Managed AM - app = rm.submitApp(1024); + app = MockRMAppSubmitter.submitWithMemory(1024, rm); // Case 2.1: AppAttemptId is null report = NMContainerStatus.newInstance( @@ -2680,7 +2684,11 @@ protected Dispatcher createDispatcher() { }; rm.start(); - RMApp app = rm.submitApp(1024, true); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm) + .withUnmanaged(true) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt() .getAppAttemptId(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestSignalContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestSignalContainer.java index c63df36cb0c..9f03a7226f9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestSignalContainer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestSignalContainer.java @@ -57,7 +57,7 @@ public void testSignalRequestDeliveryToNM() throws Exception { MockNM nm1 = rm.registerNode("h1:1234", 5000); - RMApp app = rm.submitApp(2000); + RMApp app = MockRMAppSubmitter.submitWithMemory(2000, rm); //kick the scheduling nm1.nodeHeartbeat(true); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestSubmitApplicationWithRMHA.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestSubmitApplicationWithRMHA.java index c4fa74d093b..4c862c06ff2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestSubmitApplicationWithRMHA.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestSubmitApplicationWithRMHA.java @@ -53,11 +53,18 @@ // submit the application with previous assigned applicationId // to current active rm: rm2 RMApp app1 = - rm2.submitApp(200, "", UserGroupInformation - .getCurrentUser().getShortUserName(), null, false, null, - configuration.getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, - false, false, true, appId); + MockRMAppSubmitter.submit(rm2, + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm2) + .withName("") + .withUser(UserGroupInformation + .getCurrentUser().getShortUserName()) + .withAcls(null) + .withUnmanaged(false) + .withQueue(null) + .withWaitForAccepted(false) + .withIsAppIdProvided(true) + .withApplicationId(appId) + .build()); // verify application submission verifySubmitApp(rm2, app1, appId); @@ -106,7 +113,7 @@ private void verifySubmitApp(MockRM rm, RMApp app, // Submit Application // After submission, the applicationState will be saved in RMStateStore. - RMApp app0 = rm1.submitApp(200); + RMApp app0 = MockRMAppSubmitter.submitWithMemory(200, rm1); // Do the failover explicitFailover(); @@ -137,12 +144,23 @@ private void verifySubmitApp(MockRM rm, RMApp app, // Submit Application // After submission, the applicationState will // not be saved in RMStateStore + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("") + .withUser(UserGroupInformation + .getCurrentUser().getShortUserName()) + .withAcls(null) + .withUnmanaged(false) + .withQueue(null) + .withMaxAppAttempts(configuration.getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, + YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS)) + .withCredentials(null) + .withAppType(null) + .withWaitForAccepted(false) + .withKeepContainers(false) + .build(); RMApp app0 = - rm1.submitApp(200, "", UserGroupInformation - .getCurrentUser().getShortUserName(), null, false, null, - configuration.getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, - false, false); + MockRMAppSubmitter.submit(rm1, data); // Do the failover explicitFailover(); @@ -163,11 +181,23 @@ private void verifySubmitApp(MockRM rm, RMApp app, // Application with previous applicationId // when catches the ApplicationNotFoundException RMApp app1 = - rm2.submitApp(200, "", UserGroupInformation - .getCurrentUser().getShortUserName(), null, false, null, - configuration.getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, - false, false, true, app0.getApplicationId()); + MockRMAppSubmitter.submit(rm2, + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm2) + .withName("") + .withUser(UserGroupInformation + .getCurrentUser().getShortUserName()) + .withAcls(null) + .withUnmanaged(false) + .withQueue(null) + .withMaxAppAttempts(configuration.getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, + YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS)) + .withCredentials(null) + .withAppType(null) + .withWaitForAccepted(false) + .withKeepContainers(false) + .withIsAppIdProvided(true) + .withApplicationId(app0.getApplicationId()) + .build()); verifySubmitApp(rm2, app1, app0.getApplicationId()); } @@ -183,7 +213,7 @@ public void testGetApplicationReportIdempotent() throws Exception{ // Submit Application // After submission, the applicationState will be saved in RMStateStore. - RMApp app = rm1.submitApp(200); + RMApp app = MockRMAppSubmitter.submitWithMemory(200, rm1); ApplicationReport appReport1 = rm1.getApplicationReport(app.getApplicationId()); @@ -235,7 +265,7 @@ public void testGetApplicationReportIdempotent() throws Exception{ // Submit Application // After submission, the applicationState will be saved in RMStateStore. - RMApp app0 = rm1.submitApp(200); + RMApp app0 = MockRMAppSubmitter.submitWithMemory(200, rm1); // Do the failover explicitFailover(); @@ -251,11 +281,23 @@ public void testGetApplicationReportIdempotent() throws Exception{ // check whether this application has been exist. If yes, just simply // return submitApplicationResponse. RMApp app1 = - rm2.submitApp(200, "", UserGroupInformation - .getCurrentUser().getShortUserName(), null, false, null, - configuration.getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, - false, false, true, app0.getApplicationId()); + MockRMAppSubmitter.submit(rm2, + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm2) + .withName("") + .withUser(UserGroupInformation + .getCurrentUser().getShortUserName()) + .withAcls(null) + .withUnmanaged(false) + .withQueue(null) + .withMaxAppAttempts(configuration.getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, + YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS)) + .withCredentials(null) + .withAppType(null) + .withWaitForAccepted(false) + .withKeepContainers(false) + .withIsAppIdProvided(true) + .withApplicationId(app0.getApplicationId()) + .build()); Assert.assertEquals(app1.getApplicationId(), app0.getApplicationId()); } @@ -273,12 +315,23 @@ public void testGetApplicationReportIdempotent() throws Exception{ // Submit Application // After submission, the applicationState will // not be saved in RMStateStore + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("") + .withUser(UserGroupInformation + .getCurrentUser().getShortUserName()) + .withAcls(null) + .withUnmanaged(false) + .withQueue(null) + .withMaxAppAttempts(configuration.getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, + YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS)) + .withCredentials(null) + .withAppType(null) + .withWaitForAccepted(false) + .withKeepContainers(false) + .build(); RMApp app0 = - rm1.submitApp(200, "", UserGroupInformation - .getCurrentUser().getShortUserName(), null, false, null, - configuration.getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, - false, false); + MockRMAppSubmitter.submit(rm1, data); // Do the failover explicitFailover(); @@ -294,11 +347,23 @@ public void testGetApplicationReportIdempotent() throws Exception{ // submitApplication() when failover happens during the submission process // because the submitApplication api is marked as idempotent RMApp app1 = - rm2.submitApp(200, "", UserGroupInformation - .getCurrentUser().getShortUserName(), null, false, null, - configuration.getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, - false, false, true, app0.getApplicationId()); + MockRMAppSubmitter.submit(rm2, + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm2) + .withName("") + .withUser(UserGroupInformation + .getCurrentUser().getShortUserName()) + .withAcls(null) + .withUnmanaged(false) + .withQueue(null) + .withMaxAppAttempts(configuration.getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, + YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS)) + .withCredentials(null) + .withAppType(null) + .withWaitForAccepted(false) + .withKeepContainers(false) + .withIsAppIdProvided(true) + .withApplicationId(app0.getApplicationId()) + .build()); verifySubmitApp(rm2, app1, app0.getApplicationId()); Assert.assertTrue(rm2.getRMContext().getRMApps() diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java index d503cb44fb8..894482fa276 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java @@ -170,7 +170,7 @@ public void testSchedulerRecovery() throws Exception { MockNM nm1 = new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService()); nm1.registerNode(); - RMApp app1 = rm1.submitApp(200); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(200, rm1); Resource amResources = app1.getAMResourceRequests().get(0).getCapability(); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); @@ -335,9 +335,15 @@ public void testDynamicQueueRecovery() throws Exception { // dynamic queue. rm1.getRMContext().getReservationSystem() .synchronizePlan(ReservationSystemTestUtil.reservationQ, true); - RMApp app1 = rm1.submitApp(200, "dynamicQApp", - UserGroupInformation.getCurrentUser().getShortUserName(), null, - ReservationSystemTestUtil.getReservationQueueName()); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("dynamicQApp") + .withUser(UserGroupInformation.getCurrentUser().getShortUserName()) + .withAcls(null) + .withQueue(ReservationSystemTestUtil.getReservationQueueName()) + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); Resource amResources = app1.getAMResourceRequests().get(0).getCapability(); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); @@ -618,7 +624,13 @@ public void testRMRestartWithRemovedQueue() throws Exception{ MockNM nm1 = new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService()); nm1.registerNode(); - final RMApp app1 = rm1.submitApp(1024, "app1", USER_1, null); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm1) + .withName("app1") + .withUser(USER_1) + .withAcls(null) + .build(); + final RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1,rm1, nm1); MockRM.finishAMAndVerifyAppState(app1, rm1, nm1, am1); @@ -671,12 +683,36 @@ public void testCapacitySchedulerRecovery() throws Exception { new MockNM("127.1.1.1:4321", 8192, rm1.getResourceTrackerService()); nm1.registerNode(); nm2.registerNode(); - RMApp app1_1 = rm1.submitApp(1024, "app1_1", USER_1, null, A); + MockRMAppSubmissionData data2 = + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm1) + .withName("app1_1") + .withUser(USER_1) + .withAcls(null) + .withQueue(A) + .withUnmanaged(false) + .build(); + RMApp app1_1 = MockRMAppSubmitter.submit(rm1, data2); MockAM am1_1 = MockRM.launchAndRegisterAM(app1_1, rm1, nm1); - RMApp app1_2 = rm1.submitApp(1024, "app1_2", USER_1, null, A); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm1) + .withName("app1_2") + .withUser(USER_1) + .withAcls(null) + .withQueue(A) + .withUnmanaged(false) + .build(); + RMApp app1_2 = MockRMAppSubmitter.submit(rm1, data1); MockAM am1_2 = MockRM.launchAndRegisterAM(app1_2, rm1, nm2); - RMApp app2 = rm1.submitApp(1024, "app2", USER_2, null, B); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm1) + .withName("app2") + .withUser(USER_2) + .withAcls(null) + .withQueue(B) + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm2); // clear queue metrics @@ -842,7 +878,15 @@ public void testCapacityLeafQueueBecomesParentOnRecovery() throws Exception { nm.registerNode(); // Submit an app to QueueB. - RMApp app = rm1.submitApp(1024, "app", USER_2, null, B); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm1) + .withName("app") + .withUser(USER_2) + .withAcls(null) + .withQueue(B) + .withUnmanaged(false) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm1, data); MockRM.launchAndRegisterAM(app, rm1, nm); assertEquals(rm1.getApplicationReport(app.getApplicationId()). getYarnApplicationState(), YarnApplicationState.RUNNING); @@ -890,20 +934,50 @@ public void testCapacitySchedulerQueueRemovedRecovery() throws Exception { new MockNM("127.1.1.1:4321", 8192, rm1.getResourceTrackerService()); nm1.registerNode(); nm2.registerNode(); - RMApp app1_1 = rm1.submitApp(1024, "app1_1", USER_1, null, A); + MockRMAppSubmissionData data2 = + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm1) + .withName("app1_1") + .withUser(USER_1) + .withAcls(null) + .withQueue(A) + .withUnmanaged(false) + .build(); + RMApp app1_1 = MockRMAppSubmitter.submit(rm1, data2); MockAM am1_1 = MockRM.launchAndRegisterAM(app1_1, rm1, nm1); - RMApp app1_2 = rm1.submitApp(1024, "app1_2", USER_1, null, A); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm1) + .withName("app1_2") + .withUser(USER_1) + .withAcls(null) + .withQueue(A) + .withUnmanaged(false) + .build(); + RMApp app1_2 = MockRMAppSubmitter.submit(rm1, data1); MockAM am1_2 = MockRM.launchAndRegisterAM(app1_2, rm1, nm2); - RMApp app2 = rm1.submitApp(1024, "app2", USER_2, null, B); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm1) + .withName("app2") + .withUser(USER_2) + .withAcls(null) + .withQueue(B) + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm2); assertEquals(rm1.getApplicationReport(app2.getApplicationId()). getYarnApplicationState(), YarnApplicationState.RUNNING); //Submit an app with a non existant queue to make sure it does not //cause a fatal failure in the non-recovery case - RMApp appNA = rm1.submitApp(1024, "app1_2", USER_1, null, - QUEUE_DOESNT_EXIST, false); + RMApp appNA = MockRMAppSubmitter.submit(rm1, + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm1) + .withName("app1_2") + .withUser(USER_1) + .withAcls(null) + .withQueue(QUEUE_DOESNT_EXIST) + .withWaitForAccepted(false) + .build()); // clear queue metrics rm1.clearQueueMetrics(app1_1); @@ -941,7 +1015,7 @@ public void testAMfailedBetweenRMRestart() throws Exception { MockNM nm1 = new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService()); nm1.registerNode(); - RMApp app1 = rm1.submitApp(200); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); rm2 = new MockRM(conf, rm1.getRMStateStore()); @@ -994,7 +1068,7 @@ public void testContainersNotRecoveredForCompletedApps() throws Exception { MockNM nm1 = new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService()); nm1.registerNode(); - RMApp app1 = rm1.submitApp(200); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); MockRM.finishAMAndVerifyAppState(app1, rm1, nm1, am1); @@ -1034,7 +1108,7 @@ public void testAppReregisterOnRMWorkPreservingRestart() throws Exception { nm1.registerNode(); // create app and launch the AM - RMApp app0 = rm1.submitApp(200); + RMApp app0 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am0 = MockRM.launchAM(app0, rm1, nm1); // Issuing registerAppAttempt() before and after RM restart to confirm // registerApplicationMaster() is idempotent. @@ -1061,7 +1135,7 @@ public void testAMContainerStatusWithRMRestart() throws Exception { MockNM nm1 = new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService()); nm1.registerNode(); - RMApp app1_1 = rm1.submitApp(1024); + RMApp app1_1 = MockRMAppSubmitter.submitWithMemory(1024, rm1); MockAM am1_1 = MockRM.launchAndRegisterAM(app1_1, rm1, nm1); RMAppAttempt attempt0 = app1_1.getCurrentAppAttempt(); @@ -1097,7 +1171,7 @@ public void testRecoverSchedulerAppAndAttemptSynchronously() throws Exception { nm1.registerNode(); // create app and launch the AM - RMApp app0 = rm1.submitApp(200); + RMApp app0 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am0 = MockRM.launchAndRegisterAM(app0, rm1, nm1); rm2 = new MockRM(conf, rm1.getRMStateStore()); @@ -1126,7 +1200,7 @@ public void testReleasedContainerNotRecovered() throws Exception { nm1.registerNode(); rm1.start(); - RMApp app1 = rm1.submitApp(1024); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(1024, rm1); final MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // Re-start RM @@ -1222,7 +1296,7 @@ public void testNewContainersNotAllocatedDuringSchedulerRecovery() MockNM nm1 = new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService()); nm1.registerNode(); - RMApp app1 = rm1.submitApp(200); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // Restart RM @@ -1280,7 +1354,7 @@ public void testRetriedFinishApplicationMasterRequest() nm1.registerNode(); // create app and launch the AM - RMApp app0 = rm1.submitApp(200); + RMApp app0 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am0 = MockRM.launchAM(app0, rm1, nm1); am0.registerAppAttempt(); @@ -1311,7 +1385,7 @@ public void testAppFailedToRenewTokenOnRecovery() throws Exception { MockNM nm1 = new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService()); nm1.registerNode(); - RMApp app1 = rm1.submitApp(200); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); MockRM rm2 = new TestSecurityMockRM(conf, rm1.getRMStateStore()) { @@ -1356,7 +1430,7 @@ public void testAppFailToValidateResourceRequestOnRecovery() throws Exception{ MockNM nm1 = new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService()); nm1.registerNode(); - RMApp app1 = rm1.submitApp(200); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // Change the config so that validateAndCreateResourceRequest throws @@ -1381,10 +1455,11 @@ public void testContainerCompleteMsgNotLostAfterAMFailedAndRMRestart() throws Ex // submit app with keepContainersAcrossApplicationAttempts true Resource resource = Records.newRecord(Resource.class); resource.setMemorySize(200); - RMApp app0 = rm1.submitApp(resource, "", UserGroupInformation - .getCurrentUser().getShortUserName(), null, false, null, - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS, null, null, true, true, - false, null, 0, null, true, null); + MockRMAppSubmissionData data = MockRMAppSubmissionData.Builder.createWithDefaults(resource, rm1) + .withKeepContainers(true) + .withMaxAppAttempts(YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS) + .build(); + RMApp app0 = MockRMAppSubmitter.submit(rm1, data); MockAM am0 = MockRM.launchAndRegisterAM(app0, rm1, nm1); am0.allocate("127.0.0.1", 1000, 2, new ArrayList()); @@ -1448,7 +1523,7 @@ public void testAppStateSavedButAttemptStateNotSaved() throws Exception { MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService()); nm1.registerNode(); - RMApp app1 = rm1.submitApp(200); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); MockRM.finishAMAndVerifyAppState(app1, rm1, nm1, am1); @@ -1484,7 +1559,11 @@ public void testUAMRecoveryOnRMWorkPreservingRestart() throws Exception { nm1.registerNode(); // create app and launch the UAM - RMApp app0 = rm1.submitApp(200, true); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withUnmanaged(true) + .build(); + RMApp app0 = MockRMAppSubmitter.submit(rm1, data); MockAM am0 = MockRM.launchUAM(app0, rm1, nm1); am0.registerAppAttempt(); @@ -1574,7 +1653,11 @@ public void testUnknownUserOnRecovery() throws Exception { nm1.registerNode(); // create app and launch the UAM - RMApp app0 = rm1.submitApp(200, true); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withUnmanaged(true) + .build(); + RMApp app0 = MockRMAppSubmitter.submit(rm1, data); MockAM am0 = MockRM.launchUAM(app0, rm1, nm1); am0.registerAppAttempt(); rm1.killApp(app0.getApplicationId()); @@ -1650,7 +1733,15 @@ public void testDynamicAutoCreatedQueueRecovery(String user, String queueName) rm1.getResourceTrackerService()); nm1.registerNode(); // 2. submit app to queue which is auto-created. - RMApp app1 = rm1.submitApp(200, "autoCreatedQApp", user, null, queueName); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("autoCreatedQApp") + .withUser(user) + .withAcls(null) + .withQueue(queueName) + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); Resource amResources = app1.getAMResourceRequests().get(0).getCapability(); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); @@ -1755,7 +1846,7 @@ public void testFairSchedulerCompletedAppsQueue() throws Exception { MockNM nm1 = new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService()); nm1.registerNode(); - RMApp app = rm1.submitApp(200); + RMApp app = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am1 = MockRM.launchAndRegisterAM(app, rm1, nm1); MockRM.finishAMAndVerifyAppState(app, rm1, nm1, am1); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingUnmanagedAM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingUnmanagedAM.java index c8cbfb7a48c..ca836a4636f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingUnmanagedAM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingUnmanagedAM.java @@ -76,10 +76,20 @@ protected void testUAMRestart(boolean keepContainers) throws Exception { boolean unamanged = true; int maxAttempts = 1; boolean waitForAccepted = true; - RMApp app = rm.submitApp(200, "", - UserGroupInformation.getCurrentUser().getShortUserName(), null, - unamanged, null, maxAttempts, null, null, waitForAccepted, - keepContainers); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm) + .withName("") + .withUser(UserGroupInformation.getCurrentUser().getShortUserName()) + .withAcls(null) + .withUnmanaged(unamanged) + .withQueue(null) + .withMaxAppAttempts(maxAttempts) + .withCredentials(null) + .withAppType(null) + .withWaitForAccepted(waitForAccepted) + .withKeepContainers(keepContainers) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); MockAM am = MockRM.launchUAM(app, rm, nm); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java index b55012304cf..a77df99fb25 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java @@ -27,6 +27,7 @@ import java.util.Map; import java.util.Random; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.junit.Assert; import org.apache.hadoop.conf.Configuration; @@ -444,7 +445,7 @@ private void testRMWritingMassiveHistory(MockRM rm) throws Exception { rm.start(); MockNM nm = rm.registerNode("127.0.0.1:1234", 1024 * 10100); - RMApp app = rm.submitApp(1024); + RMApp app = MockRMAppSubmitter.submitWithMemory(1024, rm); //Wait to make sure the attempt has the right state //TODO explore a better way than sleeping for a while (YARN-4929) Thread.sleep(1000); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java index dd9bcd08b49..a14130f9fd0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java @@ -21,6 +21,7 @@ import java.security.PrivilegedExceptionAction; import java.util.List; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.junit.Assert; import org.apache.hadoop.conf.Configuration; @@ -112,7 +113,7 @@ public void testAMRMDecommissioningNodes() throws Exception { MockNM nm2 = rm.registerNode("127.0.0.2:1234", 10000); rm.drainEvents(); - RMApp app1 = rm.submitApp(2000); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(2000, rm); // Trigger the scheduling so the AM gets 'launched' on nm1 nm1.nodeHeartbeat(true); @@ -148,7 +149,7 @@ public void testAMRMUnusableNodes() throws Exception { MockNM nm4 = rm.registerNode("127.0.0.4:1234", 10000); rm.drainEvents(); - RMApp app1 = rm.submitApp(2000); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(2000, rm); // Trigger the scheduling so the AM gets 'launched' on nm1 nm1.nodeHeartbeat(true); @@ -208,7 +209,7 @@ public void testAMRMUnusableNodes() throws Exception { Assert.assertEquals(NodeUpdateType.NODE_UNUSABLE, nr.getNodeUpdateType()); // registering another AM gives it the complete failed list - RMApp app2 = rm.submitApp(2000); + RMApp app2 = MockRMAppSubmitter.submitWithMemory(2000, rm); // Trigger nm2 heartbeat so that AM gets launched on it nm2.nodeHeartbeat(true); RMAppAttempt attempt2 = app2.getCurrentAppAttempt(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCResponseId.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCResponseId.java index d609d7a7694..aac46f7caa2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCResponseId.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCResponseId.java @@ -30,6 +30,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.junit.After; @@ -77,7 +78,7 @@ public void testARRMResponseId() throws Exception { MockNM nm1 = rm.registerNode("h1:1234", 5000); - RMApp app = rm.submitApp(2000); + RMApp app = MockRMAppSubmitter.submitWithMemory(2000, rm); // Trigger the scheduling so the AM gets 'launched' nm1.nodeHeartbeat(true); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java index 4f2fb1ac06d..fc954284c45 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java @@ -27,6 +27,7 @@ import com.google.common.base.Throwables; import com.google.common.collect.Lists; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; @@ -46,6 +47,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; import org.apache.hadoop.yarn.server.resourcemanager.MockMemoryRMStateStore; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.ParameterizedSchedulerTestBase; import org.apache.hadoop.yarn.server.resourcemanager.TestRMRestart; import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore; @@ -84,10 +87,21 @@ public void testAMRestartWithExistingContainers() throws Exception { MockRM rm1 = new MockRM(getConf()); rm1.start(); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("name") + .withUser("user") + .withAcls(new HashMap()) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(-1) + .withCredentials(null) + .withAppType("MAPREDUCE") + .withWaitForAccepted(false) + .withKeepContainers(true) + .build(); RMApp app1 = - rm1.submitApp(200, "name", "user", - new HashMap(), false, "default", -1, - null, "MAPREDUCE", false, true); + MockRMAppSubmitter.submit(rm1, data); MockNM nm1 = new MockNM("127.0.0.1:1234", 10240, rm1.getResourceTrackerService()); nm1.registerNode(); @@ -287,10 +301,21 @@ public void testNMTokensRebindOnAMRestart() throws Exception { MockRM rm1 = new MockRM(getConf()); rm1.start(); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("myname") + .withUser("myuser") + .withAcls(new HashMap()) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(-1) + .withCredentials(null) + .withAppType("MAPREDUCE") + .withWaitForAccepted(false) + .withKeepContainers(true) + .build(); RMApp app1 = - rm1.submitApp(200, "myname", "myuser", - new HashMap(), false, "default", -1, - null, "MAPREDUCE", false, true); + MockRMAppSubmitter.submit(rm1, data); MockNM nm1 = new MockNM("127.0.0.1:1234", 8000, rm1.getResourceTrackerService()); nm1.registerNode(); @@ -400,7 +425,7 @@ public void testShouldNotCountFailureToMaxAttemptRetry() throws Exception { MockNM nm1 = new MockNM("127.0.0.1:1234", 8000, rm1.getResourceTrackerService()); nm1.registerNode(); - RMApp app1 = rm1.submitApp(200); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(200, rm1); RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); AbstractYarnScheduler scheduler = @@ -525,7 +550,7 @@ public void testMaxAttemptOneMeansOne() throws Exception { MockNM nm1 = new MockNM("127.0.0.1:1234", 8000, rm1.getResourceTrackerService()); nm1.registerNode(); - RMApp app1 = rm1.submitApp(200); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(200, rm1); RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); AbstractYarnScheduler scheduler = @@ -566,7 +591,7 @@ public void testPreemptedAMRestartOnRMRestart() throws Exception { MockNM nm1 = new MockNM("127.0.0.1:1234", 8000, rm1.getResourceTrackerService()); nm1.registerNode(); - RMApp app1 = rm1.submitApp(200); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(200, rm1); RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); AbstractYarnScheduler scheduler = @@ -657,7 +682,7 @@ public void testRMRestartOrFailoverNotCountedForAMFailures() MockNM nm1 = new MockNM("127.0.0.1:1234", 8000, rm1.getResourceTrackerService()); nm1.registerNode(); - RMApp app1 = rm1.submitApp(200); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(200, rm1); // AM should be restarted even though max-am-attempt is 1. MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); @@ -735,7 +760,13 @@ public void testRMAppAttemptFailuresValidityInterval() throws Exception { // set window size to a larger number : 60s // we will verify the app should be failed if // two continuous attempts failed in 60s. - RMApp app = rm1.submitApp(200, 60000, false); + RMApp app = MockRMAppSubmitter.submit(rm1, + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("") + .withUser(UserGroupInformation.getCurrentUser().getShortUserName()) + .withKeepContainers(false) + .withAttemptFailuresValidityInterval((long) 60000) + .build()); MockAM am = MockRM.launchAM(app, rm1, nm1); // Fail current attempt normally @@ -756,7 +787,13 @@ public void testRMAppAttemptFailuresValidityInterval() throws Exception { ControlledClock clock = new ControlledClock(); // set window size to 10s - RMAppImpl app1 = (RMAppImpl)rm1.submitApp(200, 10000, false); + RMAppImpl app1 = (RMAppImpl) MockRMAppSubmitter.submit(rm1, + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("") + .withUser(UserGroupInformation.getCurrentUser().getShortUserName()) + .withKeepContainers(false) + .withAttemptFailuresValidityInterval((long) 10000) + .build()); app1.setSystemClock(clock); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // Fail attempt1 normally @@ -867,10 +904,21 @@ public void testAMRestartNotLostContainerCompleteMsg() throws Exception { MockRM rm1 = new MockRM(getConf()); rm1.start(); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("name") + .withUser("user") + .withAcls(new HashMap()) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(-1) + .withCredentials(null) + .withAppType("MAPREDUCE") + .withWaitForAccepted(false) + .withKeepContainers(true) + .build(); RMApp app1 = - rm1.submitApp(200, "name", "user", - new HashMap(), false, "default", -1, - null, "MAPREDUCE", false, true); + MockRMAppSubmitter.submit(rm1, data); MockNM nm1 = new MockNM("127.0.0.1:1234", 10240, rm1.getResourceTrackerService()); nm1.registerNode(); @@ -972,7 +1020,13 @@ public void testAMRestartNotLostContainerAfterAttemptFailuresValidityInterval() nm1.registerNode(); // set window size to 10s and enable keepContainers - RMAppImpl app1 = (RMAppImpl)rm1.submitApp(200, 10000, true); + RMAppImpl app1 = (RMAppImpl) MockRMAppSubmitter.submit(rm1, + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("") + .withUser(UserGroupInformation.getCurrentUser().getShortUserName()) + .withKeepContainers(true) + .withAttemptFailuresValidityInterval((long) 10000) + .build()); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); int NUM_CONTAINERS = 2; allocateContainers(nm1, am1, NUM_CONTAINERS); @@ -1070,9 +1124,20 @@ public void testContainersFromPreviousAttemptsWithRMRestart() MockNM nm2 = new MockNM(nm2Address, 4089, rm1.getResourceTrackerService()); nm2.registerNode(); - RMApp app1 = rm1.submitApp(200, "name", "user", - new HashMap<>(), false, "default", -1, - null, "MAPREDUCE", false, true); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("name") + .withUser("user") + .withAcls(new HashMap<>()) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(-1) + .withCredentials(null) + .withAppType("MAPREDUCE") + .withWaitForAccepted(false) + .withKeepContainers(true) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); allocateContainers(nm1, am1, 1); @@ -1205,7 +1270,12 @@ public void testQueueResourceDoesNotLeak() throws Exception { MockNM nm1 = new MockNM("127.0.0.1:1234", 8000, rm1.getResourceTrackerService()); nm1.registerNode(); - RMApp app1 = rm1.submitApp(200, 0, true); + + RMApp app1 = MockRMAppSubmitter.submit(rm1, + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withAttemptFailuresValidityInterval(0) + .withKeepContainers(true) + .build()); RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); allocateContainers(nm1, am1, 1); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/TestResourceProfiles.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/TestResourceProfiles.java index 9db129f32af..e1fdf0a25e4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/TestResourceProfiles.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/TestResourceProfiles.java @@ -26,6 +26,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.junit.Assert; @@ -162,7 +163,7 @@ public void testResourceProfilesInAMResponse() throws Exception { MockRM rm = new MockRM(conf); rm.start(); MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * 1024); - RMApp app1 = rm.submitApp(2048); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(2048, rm); nm1.nodeHeartbeat(true); RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId()); @@ -175,7 +176,7 @@ public void testResourceProfilesInAMResponse() throws Exception { rm = new MockRM(conf); rm.start(); nm1 = rm.registerNode("127.0.0.1:1234", 6 * 1024); - app1 = rm.submitApp(2048); + app1 = MockRMAppSubmitter.submitWithMemory(2048, rm); nm1.nodeHeartbeat(true); attempt1 = app1.getCurrentAppAttempt(); am1 = rm.sendAMLaunched(attempt1.getAppAttemptId()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestApplicationLifetimeMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestApplicationLifetimeMonitor.java index 8d409361581..8044bb612dc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestApplicationLifetimeMonitor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestApplicationLifetimeMonitor.java @@ -49,6 +49,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.TestRMRestart; import org.apache.hadoop.yarn.server.resourcemanager.TestWorkPreservingRMRestart; import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore; @@ -125,18 +127,34 @@ public void testApplicationLifetimeMonitor() Map timeouts = new HashMap(); timeouts.put(ApplicationTimeoutType.LIFETIME, 10L); - RMApp app1 = rm.submitApp(1024, appPriority, timeouts); + RMApp app1 = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm) + .withPriority(appPriority) + .withApplicationTimeouts(timeouts) + .build()); // 20L seconds timeouts.put(ApplicationTimeoutType.LIFETIME, 20L); - RMApp app2 = rm.submitApp(1024, appPriority, timeouts); + RMApp app2 = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm) + .withPriority(appPriority) + .withApplicationTimeouts(timeouts) + .build()); // user not set lifetime, so queue max lifetime will be considered. - RMApp app3 = rm.submitApp(1024, appPriority, Collections.emptyMap()); + RMApp app3 = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm) + .withPriority(appPriority) + .withApplicationTimeouts(Collections.emptyMap()) + .build()); // asc lifetime exceeds queue max lifetime timeouts.put(ApplicationTimeoutType.LIFETIME, 40L); - RMApp app4 = rm.submitApp(1024, appPriority, timeouts); + RMApp app4 = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm) + .withPriority(appPriority) + .withApplicationTimeouts(timeouts) + .build()); nm1.nodeHeartbeat(true); // Send launch Event @@ -235,7 +253,11 @@ public void testApplicationLifetimeOnRMRestart() throws Exception { Map timeouts = new HashMap(); timeouts.put(ApplicationTimeoutType.LIFETIME, appLifetime); - RMApp app1 = rm1.submitApp(200, Priority.newInstance(0), timeouts); + RMApp app1 = MockRMAppSubmitter.submit(rm1, + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withPriority(Priority.newInstance(0)) + .withApplicationTimeouts(timeouts) + .build()); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // Re-start RM @@ -319,7 +341,11 @@ public synchronized void updateApplicationStateInternal( Map timeouts = new HashMap(); timeouts.put(ApplicationTimeoutType.LIFETIME, appLifetime); - RMApp app1 = rm1.submitApp(200, Priority.newInstance(0), timeouts); + RMApp app1 = MockRMAppSubmitter.submit(rm1, + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withPriority(Priority.newInstance(0)) + .withApplicationTimeouts(timeouts) + .build()); Map updateTimeout = new HashMap(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestNodesListManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestNodesListManager.java index 9f0fc8ed849..0df295c9436 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestNodesListManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestNodesListManager.java @@ -38,6 +38,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockNodes; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.NodesListManager; import org.apache.hadoop.yarn.server.resourcemanager.NodesListManagerEvent; import org.apache.hadoop.yarn.server.resourcemanager.NodesListManagerEventType; @@ -72,12 +73,12 @@ protected Dispatcher createDispatcher() { RMNode rmnode = MockNodes.newNodeInfo(1, clusterResource); // Create killing APP - RMApp killrmApp = rm.submitApp(200); + RMApp killrmApp = MockRMAppSubmitter.submitWithMemory(200, rm); rm.killApp(killrmApp.getApplicationId()); rm.waitForState(killrmApp.getApplicationId(), RMAppState.KILLED); // Create finish APP - RMApp finshrmApp = rm.submitApp(2000); + RMApp finshrmApp = MockRMAppSubmitter.submitWithMemory(2000, rm); nm1.nodeHeartbeat(true); RMAppAttempt attempt = finshrmApp.getCurrentAppAttempt(); MockAM am = rm.sendAMLaunched(attempt.getAppAttemptId()); @@ -87,7 +88,7 @@ protected Dispatcher createDispatcher() { rm.waitForState(am.getApplicationAttemptId(), RMAppAttemptState.FINISHED); // Create submitted App - RMApp subrmApp = rm.submitApp(200); + RMApp subrmApp = MockRMAppSubmitter.submitWithMemory(200, rm); // Fire Event for NODE_USABLE nodesListManager.handle(new NodesListManagerEvent( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java index 256bd941972..9b1512d1e4d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java @@ -53,6 +53,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter; import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher; @@ -284,7 +285,7 @@ public void testExistenceOfResourceRequestInRMContainer() throws Exception { MockRM rm1 = new MockRM(conf); rm1.start(); MockNM nm1 = rm1.registerNode("unknownhost:1234", 8000); - RMApp app1 = rm1.submitApp(1024); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(1024, rm1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); ResourceScheduler scheduler = rm1.getResourceScheduler(); @@ -324,7 +325,7 @@ public void testStoreAllContainerMetrics() throws Exception { rm1.start(); MockNM nm1 = rm1.registerNode("unknownhost:1234", 8000); - RMApp app1 = rm1.submitApp(1024); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(1024, rm1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); nm1.nodeHeartbeat(am1.getApplicationAttemptId(), 1, ContainerState.RUNNING); @@ -364,7 +365,7 @@ public void testStoreOnlyAMContainerMetrics() throws Exception { rm1.start(); MockNM nm1 = rm1.registerNode("unknownhost:1234", 8000); - RMApp app1 = rm1.submitApp(1024); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(1024, rm1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); nm1.nodeHeartbeat(am1.getApplicationAttemptId(), 1, ContainerState.RUNNING); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java index b58c7a411c2..ff36d3bb118 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java @@ -53,6 +53,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockNodes; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.NMLivelinessMonitor; import org.apache.hadoop.yarn.server.resourcemanager.NodesListManager; import org.apache.hadoop.yarn.server.resourcemanager.ParameterizedSchedulerTestBase; @@ -402,7 +404,7 @@ public void testReleasedContainerIfAppAttemptisNull() throws Exception { new SchedulerApplication(null, mockAPp.getUser()); // Second app with one app attempt - RMApp app = rm1.submitApp(200); + RMApp app = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am1 = MockRM.launchAndRegisterAM(app, rm1, nm1); final ContainerId runningContainer = ContainerId.newContainerId(am1.getApplicationAttemptId(), 2); @@ -439,9 +441,21 @@ public void testContainerReleaseWithAllocationTags() throws Exception { MockNM nm1 = new MockNM("127.0.0.1:1234", 10240, rm1.getResourceTrackerService()); nm1.registerNode(); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("name") + .withUser("user") + .withAcls(new HashMap<>()) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(-1) + .withCredentials(null) + .withAppType("Test") + .withWaitForAccepted(false) + .withKeepContainers(true) + .build(); RMApp app1 = - rm1.submitApp(200, "name", "user", new HashMap<>(), false, "default", - -1, null, "Test", false, true); + MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // allocate 1 container with tag1 @@ -530,10 +544,21 @@ public void testContainerReleasedByNode() throws Exception { MockRM rm1 = new MockRM(conf); try { rm1.start(); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("name") + .withUser("user") + .withAcls(new HashMap()) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(-1) + .withCredentials(null) + .withAppType("Test") + .withWaitForAccepted(false) + .withKeepContainers(true) + .build(); RMApp app1 = - rm1.submitApp(200, "name", "user", - new HashMap(), false, "default", - -1, null, "Test", false, true); + MockRMAppSubmitter.submit(rm1, data); MockNM nm1 = new MockNM("127.0.0.1:1234", 10240, rm1.getResourceTrackerService()); nm1.registerNode(); @@ -642,10 +667,21 @@ public void testResourceRequestRestoreWhenRMContainerIsAtAllocated() MockRM rm1 = new MockRM(conf); try { rm1.start(); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("name") + .withUser("user") + .withAcls(new HashMap()) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(-1) + .withCredentials(null) + .withAppType("Test") + .withWaitForAccepted(false) + .withKeepContainers(true) + .build(); RMApp app1 = - rm1.submitApp(200, "name", "user", - new HashMap(), false, "default", - -1, null, "Test", false, true); + MockRMAppSubmitter.submit(rm1, data); MockNM nm1 = new MockNM("127.0.0.1:1234", 10240, rm1.getResourceTrackerService()); nm1.registerNode(); @@ -730,10 +766,21 @@ public void testResourceRequestRecoveryToTheRightAppAttempt() MockRM rm = new MockRM(conf); try { rm.start(); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm) + .withName("name") + .withUser("user") + .withAcls(new HashMap()) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(-1) + .withCredentials(null) + .withAppType("Test") + .withWaitForAccepted(false) + .withKeepContainers(true) + .build(); RMApp rmApp = - rm.submitApp(200, "name", "user", - new HashMap(), false, "default", -1, - null, "Test", false, true); + MockRMAppSubmitter.submit(rm, data); MockNM node = new MockNM("127.0.0.1:1234", 10240, rm.getResourceTrackerService()); node.registerNode(); @@ -988,10 +1035,21 @@ public void testContainerRecoveredByNode() throws Exception { MockRM rm1 = new MockRM(conf); try { rm1.start(); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("name") + .withUser("user") + .withAcls(new HashMap()) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(-1) + .withCredentials(null) + .withAppType("Test") + .withWaitForAccepted(false) + .withKeepContainers(true) + .build(); RMApp app1 = - rm1.submitApp(200, "name", "user", - new HashMap(), false, "default", - -1, null, "Test", false, true); + MockRMAppSubmitter.submit(rm1, data); MockNM nm1 = new MockNM("127.0.0.1:1234", 10240, rm1.getResourceTrackerService()); nm1.registerNode(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerOvercommit.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerOvercommit.java index 36b51985060..d493bce1daa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerOvercommit.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerOvercommit.java @@ -56,6 +56,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.TestResourceTrackerService.NullNodeAttributeStore; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; @@ -127,7 +129,8 @@ public void setup() throws Exception { nmId = nm.getNodeId(); // Start an AM with 2GB - RMApp app = rm.submitApp(2 * GB); + RMApp app = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(2 * GB, rm).build()); nm.nodeHeartbeat(true); attempt = app.getCurrentAppAttempt(); am = rm.sendAMLaunched(attempt.getAppAttemptId()); @@ -367,7 +370,11 @@ public void testKillMultipleContainers() throws Exception { waitMemory(scheduler, nmId, 4 * GB, 4 * GB, 200, 5 * 1000); // Start an AM with 2GB - RMApp app2 = rm.submitApp(2 * GB, "app2", "user2"); + RMApp app2 = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(2 * GB, rm) + .withName("app2") + .withUser("user2") + .build()); nm.nodeHeartbeat(true); RMAppAttempt attempt2 = app2.getCurrentAppAttempt(); MockAM am2 = rm.sendAMLaunched(attempt2.getAppAttemptId()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java index 645d6638cb4..9839ad48459 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java @@ -77,6 +77,8 @@ import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; import org.apache.hadoop.yarn.resourcetypes.ResourceTypesTestHelper; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MockRMWithAMS; import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MyContainerManager; @@ -690,7 +692,13 @@ public void testValidateResourceBlacklistRequest() throws Exception { Map acls = new HashMap(2); acls.put(ApplicationAccessType.VIEW_APP, "*"); - RMApp app = rm.submitApp(1024, "appname", "appuser", acls); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm) + .withName("appname") + .withUser("appuser") + .withAcls(acls) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); nm1.nodeHeartbeat(true); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulingWithAllocationRequestId.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulingWithAllocationRequestId.java index 8e983321136..ce6342f1f71 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulingWithAllocationRequestId.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulingWithAllocationRequestId.java @@ -28,6 +28,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.ParameterizedSchedulerTestBase; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; @@ -72,7 +74,7 @@ public void testMultipleAllocationRequestIds() throws Exception { MockNM nm1 = rm.registerNode("127.0.0.1:1234", 4 * GB); MockNM nm2 = rm.registerNode("127.0.0.2:5678", 4 * GB); - RMApp app1 = rm.submitApp(2048); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(2048, rm); // kick the scheduling nm1.nodeHeartbeat(true); RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); @@ -114,7 +116,7 @@ public void testMultipleAllocationRequestDiffPriority() throws Exception { MockNM nm1 = rm.registerNode("127.0.0.1:1234", 4 * GB); MockNM nm2 = rm.registerNode("127.0.0.2:5678", 4 * GB); - RMApp app1 = rm.submitApp(2048); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(2048, rm); // kick the scheduling nm1.nodeHeartbeat(true); RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); @@ -175,7 +177,12 @@ public void testMultipleAppsWithAllocationReqId() throws Exception { nm2.registerNode(); // submit 1st app - RMApp app1 = rm.submitApp(1 * GB, "user_0", "a1"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("user_0") + .withUser("a1") + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); // Submit app1 RR with allocationReqId = 5 @@ -190,7 +197,12 @@ public void testMultipleAppsWithAllocationReqId() throws Exception { checkAllocatedContainer(allocated.get(0), 1 * GB, nm1.getNodeId(), 5L); // Submit another application - RMApp app2 = rm.submitApp(1 * GB, "user_1", "a2"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("user_1") + .withUser("a2") + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm2); // Submit app2 RR with allocationReqId = 5 diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java index 172db0a3d6f..08cc13eed88 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java @@ -54,6 +54,8 @@ import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; @@ -824,20 +826,41 @@ public RMNodeLabelsManager createNodeLabelManager() { // Submit application to queue c where the default partition capacity is // zero - RMApp app1 = rm.submitApp(GB, "app", "user", null, "c", false); + RMApp app1 = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c") + .withWaitForAccepted(false) + .build()); rm.drainEvents(); rm.waitForState(app1.getApplicationId(), RMAppState.ACCEPTED); assertEquals(RMAppState.ACCEPTED, app1.getState()); rm.killApp(app1.getApplicationId()); - RMApp app2 = rm.submitApp(GB, "app", "user", null, "a1", false); + RMApp app2 = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withWaitForAccepted(false) + .build()); rm.drainEvents(); rm.waitForState(app2.getApplicationId(), RMAppState.ACCEPTED); assertEquals(RMAppState.ACCEPTED, app2.getState()); // Check second application is rejected and based on queue level max // application app is rejected - RMApp app3 = rm.submitApp(GB, "app", "user", null, "a1", false); + RMApp app3 = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withWaitForAccepted(false) + .build()); rm.drainEvents(); rm.waitForState(app3.getApplicationId(), RMAppState.FAILED); assertEquals(RMAppState.FAILED, app3.getState()); @@ -848,15 +871,36 @@ public RMNodeLabelsManager createNodeLabelManager() { app3.getDiagnostics().toString()); // based on Global limit of queue usert application is rejected - RMApp app11 = rm.submitApp(GB, "app", "user", null, "d", false); + RMApp app11 = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("d") + .withWaitForAccepted(false) + .build()); rm.drainEvents(); rm.waitForState(app11.getApplicationId(), RMAppState.ACCEPTED); assertEquals(RMAppState.ACCEPTED, app11.getState()); - RMApp app12 = rm.submitApp(GB, "app", "user", null, "d", false); + RMApp app12 = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("d") + .withWaitForAccepted(false) + .build()); rm.drainEvents(); rm.waitForState(app12.getApplicationId(), RMAppState.ACCEPTED); assertEquals(RMAppState.ACCEPTED, app12.getState()); - RMApp app13 = rm.submitApp(GB, "app", "user", null, "d", false); + RMApp app13 = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("d") + .withWaitForAccepted(false) + .build()); rm.drainEvents(); rm.waitForState(app13.getApplicationId(), RMAppState.FAILED); assertEquals(RMAppState.FAILED, app13.getState()); @@ -867,10 +911,24 @@ public RMNodeLabelsManager createNodeLabelManager() { app13.getDiagnostics().toString()); // based on system max limit application is rejected - RMApp app14 = rm.submitApp(GB, "app", "user2", null, "a2", false); + RMApp app14 = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("app") + .withUser("user2") + .withAcls(null) + .withQueue("a2") + .withWaitForAccepted(false) + .build()); rm.drainEvents(); rm.waitForState(app14.getApplicationId(), RMAppState.ACCEPTED); - RMApp app15 = rm.submitApp(GB, "app", "user2", null, "a2", false); + RMApp app15 = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("app") + .withUser("user2") + .withAcls(null) + .withQueue("a2") + .withWaitForAccepted(false) + .build()); rm.drainEvents(); rm.waitForState(app15.getApplicationId(), RMAppState.FAILED); assertEquals(RMAppState.FAILED, app15.getState()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java index e02a9b71cd6..5f491fc1cd2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java @@ -47,6 +47,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; @@ -155,14 +157,38 @@ public RMNodeLabelsManager createNodeLabelManager() { MockNM nm3 = rm1.registerNode("h3:1234", 10 * GB); // label = // Submit app1 with 1Gb AM resource to Queue A1 for label X - RMApp app1 = rm1.submitApp(GB, "app", "user", null, "a1", "x"); + MockRMAppSubmissionData data5 = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withAmLabel("x") + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data5); // Submit app2 with 1Gb AM resource to Queue A1 for label X - RMApp app2 = rm1.submitApp(GB, "app", "user", null, "a1", "x"); + MockRMAppSubmissionData data4 = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withAmLabel("x") + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data4); // Submit 3rd app to Queue A1 for label X, and this will be pending as // AM limit is already crossed for label X. (2GB) - RMApp pendingApp = rm1.submitApp(GB, "app", "user", null, "a1", "x"); + MockRMAppSubmissionData data3 = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withAmLabel("x") + .build(); + RMApp pendingApp = MockRMAppSubmitter.submit(rm1, data3); CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler(); LeafQueue leafQueue = (LeafQueue) cs.getQueue("a1"); @@ -185,16 +211,40 @@ public RMNodeLabelsManager createNodeLabelManager() { // Now verify the same test case in Queue C1 where label is not configured. // Submit an app to Queue C1 with empty label - RMApp app3 = rm1.submitApp(GB, "app", "user", null, "c1"); + MockRMAppSubmissionData data2 = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c1") + .withUnmanaged(false) + .build(); + RMApp app3 = MockRMAppSubmitter.submit(rm1, data2); MockRM.launchAndRegisterAM(app3, rm1, nm3); // Submit next app to Queue C1 with empty label - RMApp app4 = rm1.submitApp(GB, "app", "user", null, "c1"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c1") + .withUnmanaged(false) + .build(); + RMApp app4 = MockRMAppSubmitter.submit(rm1, data1); MockRM.launchAndRegisterAM(app4, rm1, nm3); // Submit 3rd app to Queue C1. This will be pending as Queue's am-limit // is reached. - pendingApp = rm1.submitApp(GB, "app", "user", null, "c1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c1") + .withUnmanaged(false) + .build(); + pendingApp = MockRMAppSubmitter.submit(rm1, data); leafQueue = (LeafQueue) cs.getQueue("c1"); Assert.assertNotNull(leafQueue); @@ -262,14 +312,30 @@ public RMNodeLabelsManager createNodeLabelManager() { MockNM nm3 = rm1.registerNode("h3:1234", 10 * GB); // label = // Submit app1 (2 GB) to Queue A1 and label X - RMApp app1 = rm1.submitApp(2 * GB, "app", "user", null, "a1", "x"); + MockRMAppSubmissionData data3 = + MockRMAppSubmissionData.Builder.createWithDefaults(2 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withAmLabel("x") + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data3); // This app must be activated eventhough the am-resource per-partition // limit is only for 1.5GB. MockRM.launchAndRegisterAM(app1, rm1, nm1); // Submit 2nd app to label "X" with one GB and it must be pending since // am-resource per-partition limit is crossed (1.5 GB was the limit). - rm1.submitApp(GB, "app", "user", null, "a1", "x"); + MockRMAppSubmissionData data2 = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withAmLabel("x") + .build(); + MockRMAppSubmitter.submit(rm1, data2); CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler(); LeafQueue leafQueue = (LeafQueue) cs.getQueue("a1"); @@ -283,14 +349,30 @@ public RMNodeLabelsManager createNodeLabelManager() { // to see queue level am-resource-limit is still working as expected. // Submit an app to Queue C1 with empty label (2 GB) - RMApp app3 = rm1.submitApp(2 * GB, "app", "user", null, "c1"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(2 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c1") + .withUnmanaged(false) + .build(); + RMApp app3 = MockRMAppSubmitter.submit(rm1, data1); // This app must be activated even though the am-resource per-queue // limit is only for 1.5GB MockRM.launchAndRegisterAM(app3, rm1, nm3); // Submit 2nd app to C1 (Default label, hence am-limit per-queue will be // considered). - rm1.submitApp(GB, "app", "user", null, "c1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c1") + .withUnmanaged(false) + .build(); + MockRMAppSubmitter.submit(rm1, data); leafQueue = (LeafQueue) cs.getQueue("c1"); Assert.assertNotNull(leafQueue); @@ -337,11 +419,27 @@ public RMNodeLabelsManager createNodeLabelManager() { rm1.registerNode("h3:1234", 10 * GB); // label = // Submit app1 (2 GB) to Queue A1 and label X - RMApp app1 = rm1.submitApp(2 * GB, "app", "user", null, "a1", "x"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(2 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withAmLabel("x") + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data1); // Submit 2nd app to label "X" with one GB. Since queue am-limit is 2GB, // 2nd app will be pending and first one will get activated. - RMApp pendingApp = rm1.submitApp(GB, "app", "user", null, "a1", "x"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withAmLabel("x") + .build(); + RMApp pendingApp = MockRMAppSubmitter.submit(rm1, data); CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler(); LeafQueue leafQueue = (LeafQueue) cs.getQueue("a1"); @@ -405,14 +503,30 @@ public RMNodeLabelsManager createNodeLabelManager() { rm1.registerNode("h3:1234", 10 * GB); // label = // Submit app1 with 1Gb AM resource to Queue A1 for label X for user0 - RMApp app1 = rm1.submitApp(GB, "app", user_0, null, "a1", "x"); + MockRMAppSubmissionData data3 = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm1) + .withName("app") + .withUser(user_0) + .withAcls(null) + .withQueue("a1") + .withAmLabel("x") + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data3); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // Place few allocate requests to make it an active application am1.allocate("*", 1 * GB, 15, new ArrayList(), ""); // Now submit 2nd app to Queue A1 for label X for user1 - RMApp app2 = rm1.submitApp(GB, "app", user_1, null, "a1", "x"); + MockRMAppSubmissionData data2 = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm1) + .withName("app") + .withUser(user_1) + .withAcls(null) + .withQueue("a1") + .withAmLabel("x") + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data2); MockRM.launchAndRegisterAM(app2, rm1, nm1); CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler(); @@ -426,7 +540,15 @@ public RMNodeLabelsManager createNodeLabelManager() { // Submit 3rd app to Queue A1 for label X for user1. Now user1 will have // 2 applications (2 GB resource) and user0 will have one app (1GB). - RMApp app3 = rm1.submitApp(GB, "app", user_1, null, "a1", "x"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm1) + .withName("app") + .withUser(user_1) + .withAcls(null) + .withQueue("a1") + .withAmLabel("x") + .build(); + RMApp app3 = MockRMAppSubmitter.submit(rm1, data1); MockAM am2 = MockRM.launchAndRegisterAM(app3, rm1, nm1); // Place few allocate requests to make it an active application. This is @@ -438,7 +560,15 @@ public RMNodeLabelsManager createNodeLabelManager() { // 4Gb -> 40% of label "X" in queue A1 // Since we have 2 users, 50% of 4Gb will be max for each user. Here user1 // has already crossed this 2GB limit, hence this app will be pending. - RMApp pendingApp = rm1.submitApp(GB, "app", user_1, null, "a1", "x"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm1) + .withName("app") + .withUser(user_1) + .withAcls(null) + .withQueue("a1") + .withAmLabel("x") + .build(); + RMApp pendingApp = MockRMAppSubmitter.submit(rm1, data); // Verify active applications count per user and also in queue level. Assert.assertEquals(3, leafQueue.getNumActiveApplications()); @@ -518,15 +648,39 @@ public RMNodeLabelsManager createNodeLabelManager() { MockNM nm5 = rm1.registerNode("h5:1234", 10 * GB); // label = // Submit app1 with 2Gb AM resource to Queue A1 for label Y - RMApp app1 = rm1.submitApp(2 * GB, "app", "user", null, "a1", "y"); + MockRMAppSubmissionData data4 = + MockRMAppSubmissionData.Builder.createWithDefaults(2 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withAmLabel("y") + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data4); MockRM.launchAndRegisterAM(app1, rm1, nm2); // Submit app2 with 1Gb AM resource to Queue A1 for label Y - RMApp app2 = rm1.submitApp(GB, "app", "user", null, "a1", "y"); + MockRMAppSubmissionData data3 = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withAmLabel("y") + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data3); MockRM.launchAndRegisterAM(app2, rm1, nm3); // Submit another app with 1Gb AM resource to Queue A1 for label Y - rm1.submitApp(GB, "app", "user", null, "a1", "y"); + MockRMAppSubmissionData data2 = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withAmLabel("y") + .build(); + MockRMAppSubmitter.submit(rm1, data2); CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler(); LeafQueue leafQueue = (LeafQueue) cs.getQueue("a1"); @@ -546,11 +700,27 @@ public RMNodeLabelsManager createNodeLabelManager() { Assert.assertEquals(1, leafQueue.getNumPendingApplications()); // Submit app3 with 1Gb AM resource to Queue B1 (no_label) - RMApp app3 = rm1.submitApp(GB, "app", "user", null, "b1"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + RMApp app3 = MockRMAppSubmitter.submit(rm1, data1); MockRM.launchAndRegisterAM(app3, rm1, nm5); // Submit another app with 1Gb AM resource to Queue B1 (no_label) - rm1.submitApp(GB, "app", "user", null, "b1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + MockRMAppSubmitter.submit(rm1, data); leafQueue = (LeafQueue) cs.getQueue("b1"); Assert.assertNotNull(leafQueue); @@ -810,12 +980,30 @@ public void testAMLimitByAllResources() throws Exception { Resource amResource = Resource.newInstance(GB, 1); - rm.submitApp(amResource, "app-1", userName, null, queueName); - rm.submitApp(amResource, "app-2", userName, null, queueName); + MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(amResource, rm) + .withName("app-1") + .withUser(userName) + .withAcls(null) + .withQueue(queueName) + .build()); + MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(amResource, rm) + .withName("app-2") + .withUser(userName) + .withAcls(null) + .withQueue(queueName) + .build()); // app-3 should not be activated as amLimit will be reached // for memory - rm.submitApp(amResource, "app-3", userName, null, queueName); + MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(amResource, rm) + .withName("app-3") + .withUser(userName) + .withAcls(null) + .withQueue(queueName) + .build()); Assert.assertEquals("PendingApplications should be 1", 1, queueA.getNumPendingApplications()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java index fea82e53887..cf5cc239146 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java @@ -40,6 +40,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockNodes; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl; @@ -160,7 +162,10 @@ public void testApplicationPriorityAllocation() throws Exception { Priority appPriority1 = Priority.newInstance(5); MockNM nm1 = rm.registerNode("127.0.0.1:1234", 16 * GB); - RMApp app1 = rm.submitApp(1 * GB, appPriority1); + MockRMAppSubmissionData data1 = MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withPriority(appPriority1) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data1); // kick the scheduler, 1 GB given to AM1, remaining 15GB on nm1 MockAM am1 = MockRM.launchAM(app1, rm, nm1); @@ -181,7 +186,10 @@ public void testApplicationPriorityAllocation() throws Exception { // Submit the second app App2 with priority 8 (Higher than App1) Priority appPriority2 = Priority.newInstance(8); - RMApp app2 = rm.submitApp(1 * GB, appPriority2); + MockRMAppSubmissionData data = MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withPriority(appPriority2) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm, data); // kick the scheduler, 1 GB which was free is given to AM of App2 MockAM am2 = MockRM.launchAM(app2, rm, nm1); @@ -245,7 +253,10 @@ public void testPriorityWithPendingApplications() throws Exception { Priority appPriority1 = Priority.newInstance(5); MockNM nm1 = rm.registerNode("127.0.0.1:1234", 8 * GB); - RMApp app1 = rm.submitApp(1 * GB, appPriority1); + MockRMAppSubmissionData data3 = MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withPriority(appPriority1) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data3); // kick the scheduler, 1 GB given to AM1, remaining 7GB on nm1 MockAM am1 = MockRM.launchAM(app1, rm, nm1); @@ -266,15 +277,24 @@ public void testPriorityWithPendingApplications() throws Exception { // Submit the second app App2 with priority 7 Priority appPriority2 = Priority.newInstance(7); - RMApp app2 = rm.submitApp(1 * GB, appPriority2); + MockRMAppSubmissionData data2 = MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withPriority(appPriority2) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm, data2); // Submit the third app App3 with priority 8 Priority appPriority3 = Priority.newInstance(8); - RMApp app3 = rm.submitApp(1 * GB, appPriority3); + MockRMAppSubmissionData data1 = MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withPriority(appPriority3) + .build(); + RMApp app3 = MockRMAppSubmitter.submit(rm, data1); // Submit the second app App4 with priority 6 Priority appPriority4 = Priority.newInstance(6); - RMApp app4 = rm.submitApp(1 * GB, appPriority4); + MockRMAppSubmissionData data = MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withPriority(appPriority4) + .build(); + RMApp app4 = MockRMAppSubmitter.submit(rm, data); // Only one app can run as AM resource limit restricts it. Kill app1, // If app3 (highest priority among rest) gets active, it indicates that @@ -308,7 +328,10 @@ public void testMaxPriorityValidation() throws Exception { Priority appPriority1 = Priority.newInstance(15); rm.registerNode("127.0.0.1:1234", 8 * GB); - RMApp app1 = rm.submitApp(1 * GB, appPriority1); + MockRMAppSubmissionData data = MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withPriority(appPriority1) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); // Application submission should be successful and verify priority Assert.assertEquals(app1.getApplicationSubmissionContext().getPriority(), @@ -329,7 +352,10 @@ public void testUpdatePriorityAtRuntime() throws Exception { Priority appPriority1 = Priority.newInstance(5); MockNM nm1 = rm.registerNode("127.0.0.1:1234", 16 * GB); - RMApp app1 = rm.submitApp(1 * GB, appPriority1); + MockRMAppSubmissionData data = MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withPriority(appPriority1) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); // kick the scheduler, 1 GB given to AM1, remaining 15GB on nm1 MockAM am1 = MockRM.launchAM(app1, rm, nm1); @@ -366,7 +392,10 @@ public void testUpdateInvalidPriorityAtRuntime() throws Exception { Priority appPriority1 = Priority.newInstance(5); MockNM nm1 = rm.registerNode("127.0.0.1:1234", 16 * GB); - RMApp app1 = rm.submitApp(1 * GB, appPriority1); + MockRMAppSubmissionData data = MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withPriority(appPriority1) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); // kick the scheduler, 1 GB given to AM1, remaining 15GB on nm1 MockAM am1 = MockRM.launchAM(app1, rm, nm1); @@ -414,7 +443,10 @@ public void testRMRestartWithChangeInPriority() throws Exception { nm1.registerNode(); Priority appPriority1 = Priority.newInstance(5); - RMApp app1 = rm1.submitApp(1 * GB, appPriority1); + MockRMAppSubmissionData data = MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withPriority(appPriority1) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); // kick the scheduler, 1 GB given to AM1, remaining 15GB on nm1 MockAM am1 = MockRM.launchAM(app1, rm1, nm1); @@ -469,7 +501,10 @@ public void testApplicationPriorityAllocationWithChangeInPriority() Priority appPriority1 = Priority.newInstance(5); MockNM nm1 = rm.registerNode("127.0.0.1:1234", 16 * GB); - RMApp app1 = rm.submitApp(1 * GB, appPriority1); + MockRMAppSubmissionData data1 = MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withPriority(appPriority1) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data1); // kick the scheduler, 1 GB given to AM1, remaining 15GB on nm1 MockAM am1 = MockRM.launchAM(app1, rm, nm1); @@ -491,7 +526,10 @@ public void testApplicationPriorityAllocationWithChangeInPriority() // Submit the second app App2 with priority 8 (Higher than App1) Priority appPriority2 = Priority.newInstance(8); - RMApp app2 = rm.submitApp(1 * GB, appPriority2); + MockRMAppSubmissionData data = MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withPriority(appPriority2) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm, data); // kick the scheduler, 1 GB which was free is given to AM of App2 MockAM am2 = MockRM.launchAM(app2, rm, nm1); @@ -619,13 +657,19 @@ public void testOrderOfActivatingThePriorityApplicationOnRMRestart() // App-1 with priority 5 submitted and running Priority appPriority1 = Priority.newInstance(5); - RMApp app1 = rm1.submitApp(memory, appPriority1); + MockRMAppSubmissionData data2 = MockRMAppSubmissionData.Builder.createWithDefaults(memory, rm1) + .withPriority(appPriority1) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data2); MockAM am1 = MockRM.launchAM(app1, rm1, nm1); am1.registerAppAttempt(); // App-2 with priority 6 submitted and running Priority appPriority2 = Priority.newInstance(6); - RMApp app2 = rm1.submitApp(memory, appPriority2); + MockRMAppSubmissionData data1 = MockRMAppSubmissionData.Builder.createWithDefaults(memory, rm1) + .withPriority(appPriority2) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data1); MockAM am2 = MockRM.launchAM(app2, rm1, nm1); am2.registerAppAttempt(); @@ -636,7 +680,10 @@ public void testOrderOfActivatingThePriorityApplicationOnRMRestart() // App-3 with priority 7 submitted and scheduled. But not activated since // AMResourceLimit threshold Priority appPriority3 = Priority.newInstance(7); - RMApp app3 = rm1.submitApp(memory, appPriority3); + MockRMAppSubmissionData data = MockRMAppSubmissionData.Builder.createWithDefaults(memory, rm1) + .withPriority(appPriority3) + .build(); + RMApp app3 = MockRMAppSubmitter.submit(rm1, data); rm1.drainEvents(); Assert.assertEquals(2, defaultQueue.getNumActiveApplications()); @@ -739,7 +786,10 @@ public void testUpdatePriorityOnPendingAppAndKillAttempt() throws Exception { CSQueue defaultQueue = (LeafQueue) cs.getQueue("default"); // Update priority and kill application with no resource - RMApp app1 = rm.submitApp(1024, Priority.newInstance(appPriority)); + MockRMAppSubmissionData data3 = MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm) + .withPriority(Priority.newInstance(appPriority)) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data3); Collection appsPending = ((LeafQueue) defaultQueue).getPendingApplications(); Collection activeApps = @@ -756,11 +806,20 @@ public void testUpdatePriorityOnPendingAppAndKillAttempt() throws Exception { MockNM nm1 = new MockNM("127.0.0.1:1234", 8096, rm.getResourceTrackerService()); nm1.registerNode(); - RMApp app2 = rm.submitApp(1024, Priority.newInstance(appPriority)); + MockRMAppSubmissionData data2 = MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm) + .withPriority(Priority.newInstance(appPriority)) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm, data2); Assert.assertEquals("Pending apps should be 0", 0, appsPending.size()); Assert.assertEquals("Active apps should be 1", 1, activeApps.size()); - RMApp app3 = rm.submitApp(1024, Priority.newInstance(appPriority)); - RMApp app4 = rm.submitApp(1024, Priority.newInstance(appPriority)); + MockRMAppSubmissionData data1 = MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm) + .withPriority(Priority.newInstance(appPriority)) + .build(); + RMApp app3 = MockRMAppSubmitter.submit(rm, data1); + MockRMAppSubmissionData data = MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm) + .withPriority(Priority.newInstance(appPriority)) + .build(); + RMApp app4 = MockRMAppSubmitter.submit(rm, data); Assert.assertEquals("Pending apps should be 2", 2, appsPending.size()); Assert.assertEquals("Active apps should be 1", 1, activeApps.size()); // kill app3, pending apps should reduce to 1 diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSAllocateCustomResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSAllocateCustomResource.java index b9ffb4db5be..bcb065af027 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSAllocateCustomResource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSAllocateCustomResource.java @@ -26,6 +26,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; @@ -122,7 +124,13 @@ public void testCapacitySchedulerJobWhenConfigureCustomResourceType() // submit app Resource amResource = Resources.createResource(1 * g, 1); amResource.setResourceValue(customResourceType, 1); - RMApp app1 = rm.submitApp(amResource, "app", "user", null, "a"); + RMApp app1 = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(amResource, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a") + .build()); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); // am request containers diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index 01be51c3255..31783da8ace 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -109,6 +109,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockNodes; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.NodeManager; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; @@ -1334,7 +1336,7 @@ public void testResourceOverCommit() throws Exception { MockNM nm = rm.registerNode("127.0.0.1:1234", 4 * GB); NodeId nmId = nm.getNodeId(); - RMApp app = rm.submitApp(2048); + RMApp app = MockRMAppSubmitter.submitWithMemory(2048, rm); // kick the scheduling, 2 GB given to AM1, remaining 2GB on nm nm.nodeHeartbeat(true); RMAppAttempt attempt1 = app.getCurrentAppAttempt(); @@ -1578,7 +1580,13 @@ public void testAllocateDoesNotBlockOnSchedulerLock() throws Exception { Map acls = new HashMap(2); acls.put(ApplicationAccessType.VIEW_APP, "*"); - RMApp app = rm.submitApp(1024, "appname", "appuser", acls); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm) + .withName("appname") + .withUser("appuser") + .withAcls(acls) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); nm1.nodeHeartbeat(true); @@ -1724,7 +1732,7 @@ public void testPreemptionInfo() throws Exception { nm1.registerNode(); // create app and launch the AM - RMApp app0 = rm1.submitApp(CONTAINER_MEMORY); + RMApp app0 = MockRMAppSubmitter.submitWithMemory(CONTAINER_MEMORY, rm1); MockAM am0 = MockRM.launchAM(app0, rm1, nm1); am0.registerAppAttempt(); @@ -1790,7 +1798,7 @@ public void testRecoverRequestAfterPreemption() throws Exception { MockRM rm1 = new MockRM(conf); rm1.start(); MockNM nm1 = rm1.registerNode("127.0.0.1:1234", 8000); - RMApp app1 = rm1.submitApp(1024); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(1024, rm1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler(); @@ -1870,7 +1878,15 @@ public void testMoveAppBasic() throws Exception { QueueMetrics metrics = scheduler.getRootQueueMetrics(); Assert.assertEquals(0, metrics.getAppsPending()); // submit an app - RMApp app = rm.submitApp(GB, "test-move-1", "user_0", null, "a1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("test-move-1") + .withUser("user_0") + .withAcls(null) + .withQueue("a1") + .withUnmanaged(false) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); ApplicationAttemptId appAttemptId = rm.getApplicationReport(app.getApplicationId()) .getCurrentApplicationAttemptId(); @@ -1942,8 +1958,20 @@ public void testMoveAppPendingMetrics() throws Exception { Assert.assertEquals(0, metrics.getAppsPending()); // submit two apps in a1 - RMApp app1 = rm.submitApp(GB, "test-move-1", "user_0", null, "a1"); - RMApp app2 = rm.submitApp(GB, "test-move-2", "user_0", null, "a1"); + RMApp app1 = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("test-move-1") + .withUser("user_0") + .withAcls(null) + .withQueue("a1") + .build()); + RMApp app2 = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("test-move-2") + .withUser("user_0") + .withAcls(null) + .withQueue("a1") + .build()); appsInA1 = scheduler.getAppsInQueue("a1"); appsInB1 = scheduler.getAppsInQueue("b1"); @@ -1952,7 +1980,13 @@ public void testMoveAppPendingMetrics() throws Exception { assertEquals(2, metrics.getAppsPending()); // submit one app in b1 - RMApp app3 = rm.submitApp(GB, "test-move-2", "user_0", null, "b1"); + RMApp app3 = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("test-move-2") + .withUser("user_0") + .withAcls(null) + .withQueue("b1") + .build()); appsInA1 = scheduler.getAppsInQueue("a1"); appsInB1 = scheduler.getAppsInQueue("b1"); @@ -1996,7 +2030,15 @@ public void testMoveAppSameParent() throws Exception { (AbstractYarnScheduler) rm.getResourceScheduler(); // submit an app - RMApp app = rm.submitApp(GB, "test-move-1", "user_0", null, "a1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("test-move-1") + .withUser("user_0") + .withAcls(null) + .withQueue("a1") + .withUnmanaged(false) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); ApplicationAttemptId appAttemptId = rm.getApplicationReport(app.getApplicationId()) .getCurrentApplicationAttemptId(); @@ -2512,7 +2554,15 @@ public void testMoveAllApps() throws Exception { (AbstractYarnScheduler) rm.getResourceScheduler(); // submit an app - RMApp app = rm.submitApp(GB, "test-move-1", "user_0", null, "a1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("test-move-1") + .withUser("user_0") + .withAcls(null) + .withQueue("a1") + .withUnmanaged(false) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); ApplicationAttemptId appAttemptId = rm.getApplicationReport(app.getApplicationId()) .getCurrentApplicationAttemptId(); @@ -2574,7 +2624,15 @@ public void testMoveAllAppsInvalidDestination() throws Exception { YarnScheduler scheduler = rm.getResourceScheduler(); // submit an app - RMApp app = rm.submitApp(GB, "test-move-1", "user_0", null, "a1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("test-move-1") + .withUser("user_0") + .withAcls(null) + .withQueue("a1") + .withUnmanaged(false) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); ApplicationAttemptId appAttemptId = rm.getApplicationReport(app.getApplicationId()) .getCurrentApplicationAttemptId(); @@ -2632,7 +2690,15 @@ public void testMoveAllAppsInvalidSource() throws Exception { YarnScheduler scheduler = rm.getResourceScheduler(); // submit an app - RMApp app = rm.submitApp(GB, "test-move-1", "user_0", null, "a1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("test-move-1") + .withUser("user_0") + .withAcls(null) + .withQueue("a1") + .withUnmanaged(false) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); ApplicationAttemptId appAttemptId = rm.getApplicationReport(app.getApplicationId()) .getCurrentApplicationAttemptId(); @@ -2890,7 +2956,15 @@ public void testKillAllAppsInQueue() throws Exception { (AbstractYarnScheduler) rm.getResourceScheduler(); // submit an app - RMApp app = rm.submitApp(GB, "test-move-1", "user_0", null, "a1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("test-move-1") + .withUser("user_0") + .withAcls(null) + .withQueue("a1") + .withUnmanaged(false) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); ApplicationAttemptId appAttemptId = rm.getApplicationReport(app.getApplicationId()) .getCurrentApplicationAttemptId(); @@ -2935,7 +3009,15 @@ public void testKillAllAppsInvalidSource() throws Exception { YarnScheduler scheduler = rm.getResourceScheduler(); // submit an app - RMApp app = rm.submitApp(GB, "test-move-1", "user_0", null, "a1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("test-move-1") + .withUser("user_0") + .withAcls(null) + .withQueue("a1") + .withUnmanaged(false) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); ApplicationAttemptId appAttemptId = rm.getApplicationReport(app.getApplicationId()) .getCurrentApplicationAttemptId(); @@ -2996,7 +3078,7 @@ public void testAppReservationWithDominantResourceCalculator() throws Exception MockNM nm2 = rm.registerNode("127.0.0.1:1235", 10 * GB, 4); rm.registerNode("127.0.0.1:1236", 10 * GB, 4); - RMApp app1 = rm.submitApp(1024); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(1024, rm); // kick the scheduling nm1.nodeHeartbeat(true); RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); @@ -3568,7 +3650,15 @@ public void testSchedulerKeyGarbageCollection() throws Exception { nm3.registerNode(); nm4.registerNode(); - RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); ApplicationAttemptId attemptId = app1.getCurrentAppAttempt().getAppAttemptId(); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm2); @@ -3719,7 +3809,15 @@ public void testHierarchyQueuesCurrentLimits() throws Exception { new MockNM("127.0.0.1:1234", 100 * GB, rm1.getResourceTrackerService()); nm1.registerNode(); - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "b1"); + MockRMAppSubmissionData data2 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data2); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); waitContainerAllocated(am1, 1 * GB, 1, 2, rm1, nm1); @@ -3729,7 +3827,15 @@ public void testHierarchyQueuesCurrentLimits() throws Exception { Assert.assertEquals(69 * GB, am1.doHeartbeat().getAvailableResources().getMemorySize()); - RMApp app2 = rm1.submitApp(1 * GB, "app", "user", null, "b2"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b2") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data1); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1); // Allocate 5 containers, each one is 8 GB in am2 (40 GB in total) @@ -3747,7 +3853,15 @@ public void testHierarchyQueuesCurrentLimits() throws Exception { // Now we submit app3 to a1 (in higher level hierarchy), to see if headroom // of app1 (in queue b1) updated correctly - RMApp app3 = rm1.submitApp(1 * GB, "app", "user", null, "a1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withUnmanaged(false) + .build(); + RMApp app3 = MockRMAppSubmitter.submit(rm1, data); MockAM am3 = MockRM.launchAndRegisterAM(app3, rm1, nm1); // Allocate 3 containers, each one is 8 GB in am3 (24 GB in total) @@ -3800,12 +3914,28 @@ public void testParentQueueMaxCapsAreRespected() throws Exception { nm1.registerNode(); // Launch app1 in a1, resource usage is 1GB (am) + 4GB * 2 = 9GB - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "a1"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); waitContainerAllocated(am1, 4 * GB, 2, 2, rm1, nm1); // Try to launch app2 in a2, asked 2GB, should success - RMApp app2 = rm1.submitApp(2 * GB, "app", "user", null, "a2"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(2 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a2") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1); try { // Try to allocate a container, a's usage=11G/max=12 @@ -3848,11 +3978,27 @@ protected RMNodeLabelsManager createNodeLabelManager() { nm2.registerNode(); // Launch app1 in queue=a1 - RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "a1"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm2); // Launch app2 in queue=b1 - RMApp app2 = rm.submitApp(8 * GB, "app", "user", null, "b1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(8 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm2); // am1 asks for 8 * 1GB container for no label @@ -3968,7 +4114,15 @@ public void testAMUsedResource() throws Exception { amMemory < minAllocMb); Resource minAllocResource = Resource.newInstance(minAllocMb, 1); String queueName = "a1"; - RMApp rmApp = rm.submitApp(amMemory, "app-1", "user_0", null, queueName); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(amMemory, rm) + .withName("app-1") + .withUser("user_0") + .withAcls(null) + .withQueue(queueName) + .withUnmanaged(false) + .build(); + RMApp rmApp = MockRMAppSubmitter.submit(rm, data); assertEquals("RMApp does not containes minimum allocation", minAllocResource, rmApp.getAMResourceRequests().get(0).getCapability()); @@ -4237,7 +4391,15 @@ protected RMNodeLabelsManager createNodeLabelManager() { nm1.registerNode(); // Launch app1 in queue=a1 - RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "a1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); // Allocate two more containers @@ -4344,9 +4506,23 @@ public Boolean get() { } }, 100, 60 * 1000); - rm.submitApp(amResource1, "app-1", userName, null, queueName); - - rm.submitApp(amResource2, "app-2", userName, null, queueName); + MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(amResource1, rm) + .withResource(amResource1) + .withName("app-1") + .withUser(userName) + .withAcls(null) + .withQueue(queueName) + .build()); + + MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(amResource2, rm) + .withResource(amResource2) + .withName("app-2") + .withUser(userName) + .withAcls(null) + .withQueue(queueName) + .build()); // When AM limit is exceeded, 1 applications will be activated.Rest all // applications will be in pending @@ -4532,7 +4708,7 @@ public void testSchedulingOnRemovedNode() throws Exception { MockRM rm = new MockRM(conf); rm.start(); - RMApp app = rm.submitApp(100); + RMApp app = MockRMAppSubmitter.submitWithMemory(100, rm); rm.drainEvents(); MockNM nm1 = rm.registerNode("127.0.0.1:1234", 10240, 10); @@ -4814,7 +4990,15 @@ protected RMNodeLabelsManager createNodeLabelManager() { nm1.registerNode(); // Launch app1 in queue=a1 - RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "a"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); // Got one offswitch request and offswitch allocation MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); @@ -5148,7 +5332,15 @@ public void testQueueMappingWithCurrentUserQueueMappingForaGroup() throws CapacityScheduler cs = ((CapacityScheduler) rm.getResourceScheduler()); cs.start(); - RMApp app = rm.submitApp(GB, "appname", "a1", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("appname") + .withUser("a1") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); List appsInA1 = cs.getAppsInQueue("a1"); assertEquals(1, appsInA1.size()); } @@ -5265,7 +5457,12 @@ public void testClearRequestsBeforeApplyTheProposal() final MockNM nm = rm.registerNode("h1:1234", 200 * GB); // submit app - final RMApp app = rm.submitApp(200, "app", "user"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm) + .withName("app") + .withUser("user") + .build(); + final RMApp app = MockRMAppSubmitter.submit(rm, data); MockRM.launchAndRegisterAM(app, rm, nm); // spy capacity scheduler to handle CapacityScheduler#apply @@ -5329,7 +5526,12 @@ public void testContainerAllocationLocalitySkipped() throws Exception { // submit app and request resource // container2 is larger than nm1 total resource, will trigger locality skip - final RMApp app = rm.submitApp(1 * GB, "app", "user"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .build(); + final RMApp app = MockRMAppSubmitter.submit(rm, data); final MockAM am = MockRM.launchAndRegisterAM(app, rm, nm1); am.addRequests(new String[] {"*"}, 5 * GB, 1, 1, 2); am.schedule(); @@ -5391,19 +5593,51 @@ public void testMoveAppWithActiveUsersWithOnlyPendingApps() throws Exception { MockNM nm1 = rm.registerNode("h1:1234", 16 * GB); // submit an app - RMApp app = rm.submitApp(GB, "test-move-1", "u1", null, "a1"); + MockRMAppSubmissionData data3 = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("test-move-1") + .withUser("u1") + .withAcls(null) + .withQueue("a1") + .withUnmanaged(false) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data3); MockAM am1 = MockRM.launchAndRegisterAM(app, rm, nm1); ApplicationAttemptId appAttemptId = rm.getApplicationReport(app.getApplicationId()) .getCurrentApplicationAttemptId(); - RMApp app2 = rm.submitApp(1 * GB, "app", "u2", null, "a1"); + MockRMAppSubmissionData data2 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("u2") + .withAcls(null) + .withQueue("a1") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm, data2); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm1); - RMApp app3 = rm.submitApp(1 * GB, "app", "u3", null, "a1"); - - RMApp app4 = rm.submitApp(1 * GB, "app", "u4", null, "a1"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("u3") + .withAcls(null) + .withQueue("a1") + .withUnmanaged(false) + .build(); + RMApp app3 = MockRMAppSubmitter.submit(rm, data1); + + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("u4") + .withAcls(null) + .withQueue("a1") + .withUnmanaged(false) + .build(); + RMApp app4 = MockRMAppSubmitter.submit(rm, data); // Each application asks 50 * 1GB containers am1.allocate("*", 1 * GB, 50, null); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java index 67c504d3df1..1de0a43904c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java @@ -33,11 +33,12 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; -import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEvent; @@ -146,8 +147,18 @@ public RMNodeLabelsManager createNodeLabelManager() { int totalAsked = 3 * GB; // 3 AMs for (int i = 0; i < 3; i++) { - RMApp rmApp = rm.submitApp(1024, "app", "user", null, false, - Character.toString((char) (i % 34 + 97)), 1, null, null, false); + RMApp rmApp = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withUnmanaged(false) + .withQueue(Character.toString((char) (i % 34 + 97))) + .withMaxAppAttempts(1) + .withCredentials(null) + .withAppType(null) + .withWaitForAccepted(false) + .build()); MockAM am = MockRM.launchAMWhenAsyncSchedulingEnabled(rmApp, rm); am.registerAppAttempt(); ams.add(am); @@ -220,8 +231,20 @@ public void testCommitProposalForFailedAppAttempt() SchedulerNode sn2 = scheduler.getSchedulerNode(nm2.getNodeId()); // launch app - RMApp app = rm.submitApp(200, "app", "user", null, false, "default", - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS, null, null, true, true); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS) + .withCredentials(null) + .withAppType(null) + .withWaitForAccepted(true) + .withKeepContainers(true) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); MockAM am = MockRM.launchAndRegisterAM(app, rm, nm1); FiCaSchedulerApp schedulerApp = scheduler.getApplicationAttempt(am.getApplicationAttemptId()); @@ -313,10 +336,26 @@ public void testCommitOutdatedReservedProposal() throws Exception { ((CapacityScheduler) scheduler).getSchedulerNode(nm2.getNodeId()); // submit app1, am1 is running on nm1 - RMApp app = rm.submitApp(200, "app", "user", null, "default"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data1); final MockAM am = MockRM.launchAndRegisterAM(app, rm, nm1); // submit app2, am2 is running on nm1 - RMApp app2 = rm.submitApp(200, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm, data); final MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm1); // allocate and launch 2 containers for app1 @@ -450,8 +489,20 @@ public void testNodeResourceOverAllocated() SchedulerNode sn1 = scheduler.getSchedulerNode(nm1.getNodeId()); // launch app - RMApp app = rm.submitApp(200, "app", "user", null, false, "default", - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS, null, null, true, true); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS) + .withCredentials(null) + .withAppType(null) + .withWaitForAccepted(true) + .withKeepContainers(true) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); MockAM am = MockRM.launchAndRegisterAM(app, rm, nm1); FiCaSchedulerApp schedulerApp = scheduler.getApplicationAttempt(am.getApplicationAttemptId()); @@ -532,8 +583,18 @@ public RMNodeLabelsManager createNodeLabelManager() { keepNMHeartbeat(nms, heartbeatInterval); for (int i = 0; i < 3; i++) { - RMApp rmApp = rm.submitApp(1024, "app", "user", null, false, - Character.toString((char) (i % 34 + 97)), 1, null, null, false); + RMApp rmApp = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withUnmanaged(false) + .withQueue(Character.toString((char) (i % 34 + 97))) + .withMaxAppAttempts(1) + .withCredentials(null) + .withAppType(null) + .withWaitForAccepted(false) + .build()); MockAM am = MockRM.launchAMWhenAsyncSchedulingEnabled(rmApp, rm); am.registerAppAttempt(); ams.add(am); @@ -632,8 +693,20 @@ public void testCommitDuplicatedAllocateFromReservedProposals() SchedulerNode sn1 = cs.getSchedulerNode(nm1.getNodeId()); // launch app - RMApp app = rm.submitApp(1 * GB, "app", "user", null, false, "default", - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS, null, null, true, true); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS) + .withCredentials(null) + .withAppType(null) + .withWaitForAccepted(true) + .withKeepContainers(true) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); MockAM am = MockRM.launchAndRegisterAM(app, rm, nm1); FiCaSchedulerApp schedulerApp = cs.getApplicationAttempt(am.getApplicationAttemptId()); @@ -720,7 +793,15 @@ public void testReleaseOutdatedReservedContainer() throws Exception { SchedulerNode sn3 = cs.getSchedulerNode(nm3.getNodeId()); // launch another app to queue, AM container should be launched in nm1 - RMApp app1 = rm1.submitApp(4 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(4 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); Resource allocateResource = Resources.createResource(5 * GB); am1.allocate("*", (int) allocateResource.getMemorySize(), 3, 0, @@ -771,13 +852,37 @@ public void testCommitProposalsForUnusableNode() throws Exception { SchedulerNode sn1 = cs.getSchedulerNode(nm1.getNodeId()); // launch app1-am on nm1 - RMApp app1 = rm.submitApp(1 * GB, "app1", "user", null, false, "default", - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS, null, null, true, true); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app1") + .withUser("user") + .withAcls(null) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS) + .withCredentials(null) + .withAppType(null) + .withWaitForAccepted(true) + .withKeepContainers(true) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); // launch app2-am on nm2 - RMApp app2 = rm.submitApp(1 * GB, "app2", "user", null, false, "default", - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS, null, null, true, true); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app2") + .withUser("user") + .withAcls(null) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS) + .withCredentials(null) + .withAppType(null) + .withWaitForAccepted(true) + .withKeepContainers(true) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm2); // app2 asks 1 * 8G container @@ -865,8 +970,20 @@ public void testReturnNullWhenGetSchedulerContainer() throws Exception { SchedulerNode sn2 = cs.getSchedulerNode(nm2.getNodeId()); // launch app1-am on nm1 - RMApp app1 = rm.submitApp(1 * GB, "app1", "user", null, false, "default", - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS, null, null, true, true); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app1") + .withUser("user") + .withAcls(null) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS) + .withCredentials(null) + .withAppType(null) + .withWaitForAccepted(true) + .withKeepContainers(true) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); // app2 asks 1 * 1G container diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java index 8e68984204a..206ebf90bfe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java @@ -24,7 +24,6 @@ import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; -import org.apache.hadoop.security.GroupMappingServiceProvider; import org.apache.hadoop.security.ShellBasedUnixGroupsMapping; import org.apache.hadoop.security.TestGroupsCaching; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; @@ -39,6 +38,8 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels .NullRMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; @@ -64,8 +65,6 @@ .AppAttemptAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event .SchedulerEvent; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair - .SimpleGroupsMapping; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.util.YarnVersionInfo; @@ -469,8 +468,15 @@ protected ApplicationId submitApp(MockRM rm, CSQueue parentQueue, CapacityScheduler capacityScheduler = (CapacityScheduler) rm.getResourceScheduler(); // submit an app - RMApp rmApp = rm.submitApp(GB, "test-auto-queue-activation", user, null, - leafQueueName); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("test-auto-queue-activation") + .withUser(user) + .withAcls(null) + .withQueue(leafQueueName) + .withUnmanaged(false) + .build(); + RMApp rmApp = MockRMAppSubmitter.submit(rm, data); // check preconditions List appsInParentQueue = @@ -564,9 +570,15 @@ protected ApplicationAttemptId submitApp(CapacityScheduler newCS, String user, protected RMApp submitApp(String user, String queue, String nodeLabel) throws Exception { - RMApp app = mockRM.submitApp(GB, - "test-auto-queue-creation" + RandomUtils.nextInt(0, 100), user, null, - queue, nodeLabel); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, mockRM) + .withName("test-auto-queue-creation" + RandomUtils.nextInt(0, 100)) + .withUser(user) + .withAcls(null) + .withQueue(queue) + .withAmLabel(nodeLabel) + .build(); + RMApp app = MockRMAppSubmitter.submit(mockRM, data); Assert.assertEquals(app.getAmNodeLabelExpression(), nodeLabel); // check preconditions List appsInC = cs.getAppsInQueue(PARENT_QUEUE); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java index 5f32e0ddff4..e30e115b052 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java @@ -31,6 +31,8 @@ import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.resourcemanager.MockNodes; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; import org.apache.hadoop.yarn.server.resourcemanager.placement @@ -70,7 +72,6 @@ import org.junit.Test; import java.io.IOException; -import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -171,11 +172,25 @@ public void testReinitializeStoppedAutoCreatedLeafQueue() throws Exception { // submit an app - RMApp app1 = mockRM.submitApp(GB, "test-auto-queue-creation-1", USER0, - null, USER0); - - RMApp app2 = mockRM.submitApp(GB, "test-auto-queue-creation-2", USER1, - null, USER1); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, mockRM) + .withName("test-auto-queue-creation-1") + .withUser(USER0) + .withAcls(null) + .withQueue(USER0) + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(mockRM, data1); + + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, mockRM) + .withName("test-auto-queue-creation-2") + .withUser(USER1) + .withAcls(null) + .withQueue(USER1) + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(mockRM, data); // check preconditions List appsInC = cs.getAppsInQueue(PARENT_QUEUE); assertEquals(2, appsInC.size()); @@ -350,8 +365,14 @@ public void testAutoCreateLeafQueueFailsWithNoQueueMapping() // submit an app under a different queue name which does not exist // and queue mapping does not exist for this user - RMApp app = mockRM.submitApp(GB, "app", INVALID_USER, null, INVALID_USER, - false); + RMApp app = MockRMAppSubmitter.submit(mockRM, + MockRMAppSubmissionData.Builder.createWithDefaults(GB, mockRM) + .withName("app") + .withUser(INVALID_USER) + .withAcls(null) + .withQueue(INVALID_USER) + .withWaitForAccepted(false) + .build()); mockRM.drainEvents(); mockRM.waitForState(app.getApplicationId(), RMAppState.FAILED); assertEquals(RMAppState.FAILED, app.getState()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java index 104145ec122..accafce8bce 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java @@ -29,6 +29,8 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationConstants; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; @@ -168,7 +170,15 @@ public void testRemoveQueue() throws Exception { a1.setEntitlement(new QueueEntitlement(A1_CAPACITY / 100, 1f)); // submit an app - RMApp app = rm.submitApp(GB, "test-move-1", "user_0", null, "a1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("test-move-1") + .withUser("user_0") + .withAcls(null) + .withQueue("a1") + .withUnmanaged(false) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); // check preconditions List appsInA1 = cs.getAppsInQueue("a1"); assertEquals(1, appsInA1.size()); @@ -203,7 +213,15 @@ public void testMoveAppToPlanQueue() throws Exception { CapacityScheduler scheduler = (CapacityScheduler) rm.getResourceScheduler(); // submit an app - RMApp app = rm.submitApp(GB, "test-move-1", "user_0", null, "b1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("test-move-1") + .withUser("user_0") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); ApplicationAttemptId appAttemptId = rm.getApplicationReport(app.getApplicationId()) .getCurrentApplicationAttemptId(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerLazyPreemption.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerLazyPreemption.java index a5d353909d9..f6857059fb8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerLazyPreemption.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerLazyPreemption.java @@ -25,6 +25,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.monitor.SchedulingMonitor; import org.apache.hadoop.yarn.server.resourcemanager.monitor.SchedulingMonitorManager; import org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy; @@ -93,7 +95,15 @@ public void testSimplePreemption() throws Exception { RMNode rmNode2 = rm1.getRMContext().getRMNodes().get(nm2.getNodeId()); // launch an app to queue, AM container should be launched in nm1 - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "a"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); am1.allocate("*", 1 * GB, 7, new ArrayList()); @@ -110,7 +120,15 @@ public void testSimplePreemption() throws Exception { Assert.assertEquals(7, schedulerApp1.getLiveContainers().size()); // Submit app2 to queue-c and asks for a 1G container for AM - RMApp app2 = rm1.submitApp(1 * GB, "app", "user", null, "c"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm2); // NM1/NM2 has available resource = 0G @@ -184,7 +202,15 @@ public void testPreemptionConsidersNodeLocalityDelay() RMNode rmNode2 = rm1.getRMContext().getRMNodes().get(nm2.getNodeId()); // launch an app to queue, AM container should be launched in nm1 - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "a"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); am1.allocate("*", 1 * GB, 6, new ArrayList()); @@ -201,7 +227,15 @@ public void testPreemptionConsidersNodeLocalityDelay() Assert.assertEquals(7, schedulerApp1.getLiveContainers().size()); // Submit app2 to queue-c and asks for a 1G container for AM - RMApp app2 = rm1.submitApp(1 * GB, "app", "user", null, "c"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm2); // NM1/NM2 has available resource = 0G @@ -277,7 +311,15 @@ public void testPreemptionConsidersHardNodeLocality() RMNode rmNode2 = rm1.getRMContext().getRMNodes().get(nm2.getNodeId()); // launch an app to queue, AM container should be launched in nm1 - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "a"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); am1.allocate("*", 1 * GB, 6, new ArrayList()); @@ -296,7 +338,15 @@ public void testPreemptionConsidersHardNodeLocality() Assert.assertEquals(7, schedulerApp1.getLiveContainers().size()); // Submit app2 to queue-c and asks for a 1G container for AM - RMApp app2 = rm1.submitApp(1 * GB, "app", "user", null, "c"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm2); // NM1/NM2 has available resource = 0G @@ -381,7 +431,15 @@ public void testPreemptionPolicyShouldRespectAlreadyMarkedKillableContainers() RMNode rmNode1 = rm1.getRMContext().getRMNodes().get(nm1.getNodeId()); // launch an app to queue, AM container should be launched in nm1 - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "a"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); am1.allocate("*", 1 * GB, 6, new ArrayList()); @@ -397,7 +455,15 @@ public void testPreemptionPolicyShouldRespectAlreadyMarkedKillableContainers() Assert.assertEquals(7, schedulerApp1.getLiveContainers().size()); // Submit app2 to queue-c and asks for a 1G container for AM - RMApp app2 = rm1.submitApp(1 * GB, "app", "user", null, "c"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1); // NM1 has available resource = 0G @@ -484,7 +550,15 @@ public void testPreemptionPolicyCleanupKillableContainersWhenNoPreemptionNeeded( RMNode rmNode1 = rm1.getRMContext().getRMNodes().get(nm1.getNodeId()); // launch an app to queue, AM container should be launched in nm1 - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "a"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); am1.allocate("*", 1 * GB, 6, new ArrayList()); @@ -500,7 +574,15 @@ public void testPreemptionPolicyCleanupKillableContainersWhenNoPreemptionNeeded( Assert.assertEquals(7, schedulerApp1.getLiveContainers().size()); // Submit app2 to queue-c and asks for a 1G container for AM - RMApp app2 = rm1.submitApp(1 * GB, "app", "user", null, "c"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1); // NM1 has available resource = 0G @@ -575,7 +657,15 @@ public void testPreemptionConsidersUserLimit() RMNode rmNode2 = rm1.getRMContext().getRMNodes().get(nm2.getNodeId()); // launch an app to queue, AM container should be launched in nm1 - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "a"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); am1.allocate("*", 1 * GB, 6, new ArrayList()); @@ -592,7 +682,15 @@ public void testPreemptionConsidersUserLimit() Assert.assertEquals(7, schedulerApp1.getLiveContainers().size()); // Submit app2 to queue-c and asks for a 1G container for AM - RMApp app2 = rm1.submitApp(1 * GB, "app", "user", null, "c"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm2); // NM1/NM2 has available resource = 0G diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerMultiNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerMultiNodes.java index 0e295765e84..a7ed7bf39fb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerMultiNodes.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerMultiNodes.java @@ -33,6 +33,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode; @@ -120,7 +122,15 @@ public void testMultiNodeSorterForSchedulingWithOrdering() throws Exception { .getNodesPerPartition(""); Assert.assertEquals(4, nodes.size()); - RMApp app1 = rm.submitApp(2048, "app-1", "user1", null, "default"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(2048, rm) + .withName("app-1") + .withUser("user1") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); SchedulerNodeReport reportNm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); @@ -134,7 +144,15 @@ public void testMultiNodeSorterForSchedulingWithOrdering() throws Exception { // Hence forcefully recompute nodes. sorter.reSortClusterNodes(); - RMApp app2 = rm.submitApp(1024, "app-2", "user2", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm) + .withName("app-2") + .withUser("user2") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm2); SchedulerNodeReport reportNm2 = rm.getResourceScheduler().getNodeReport(nm2.getNodeId()); @@ -184,11 +202,23 @@ public void testExcessReservationWillBeUnreserved() throws Exception { MockNM nm2 = rm1.registerNode("h2:1234", 8 * GB); // launch an app to queue, AM container should be launched in nm1 - RMApp app1 = rm1.submitApp(5 * GB, "app", "user", null, "default"); + RMApp app1 = MockRMAppSubmitter.submit(rm1, + MockRMAppSubmissionData.Builder.createWithDefaults(5 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .build()); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // launch another app to queue, AM container should be launched in nm2 - RMApp app2 = rm1.submitApp(5 * GB, "app", "user", null, "default"); + RMApp app2 = MockRMAppSubmitter.submit(rm1, + MockRMAppSubmissionData.Builder.createWithDefaults(5 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .build()); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm2); CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler(); @@ -263,11 +293,23 @@ public void testAllocateForReservedContainer() throws Exception { MockNM nm2 = rm1.registerNode("h2:1234", 8 * GB); // launch an app to queue, AM container should be launched in nm1 - RMApp app1 = rm1.submitApp(5 * GB, "app", "user", null, "default"); + RMApp app1 = MockRMAppSubmitter.submit(rm1, + MockRMAppSubmissionData.Builder.createWithDefaults(5 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .build()); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // launch another app to queue, AM container should be launched in nm2 - RMApp app2 = rm1.submitApp(5 * GB, "app", "user", null, "default"); + RMApp app2 = MockRMAppSubmitter.submit(rm1, + MockRMAppSubmissionData.Builder.createWithDefaults(5 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .build()); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm2); CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java index e239191e5e3..4841626ec7c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java @@ -38,6 +38,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; @@ -220,7 +222,15 @@ public RMNodeLabelsManager createNodeLabelManager() { ContainerId containerId; // launch an app to queue a1 (label = x), and check all container will // be allocated in h1 - RMApp app1 = rm.submitApp(GB, "app", "user", null, "a"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm3); ApplicationResourceUsageReport appResourceUsageReport = rm.getResourceScheduler().getAppResourceUsageReport( @@ -285,7 +295,15 @@ public RMNodeLabelsManager createNodeLabelManager() { // launch an app to queue a1 (label = x), and check all container will // be allocated in h1 - RMApp app1 = rm.submitApp(GB, "app", "user", null, "a"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm3); // request a container. @@ -446,7 +464,15 @@ public RMNodeLabelsManager createNodeLabelManager() { MockNM nm4 = rm.registerNode("h4:1234", 4096 * 2); // launch an app to queue a1 (label = x), and check all container will // be allocated in h1 - RMApp app1 = rm.submitApp(GB, "app", "user", null, "a1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm3); am1.allocate("*", GB, 1, new ArrayList(), "x"); ContainerId container1 = @@ -524,7 +550,15 @@ public RMNodeLabelsManager createNodeLabelManager() { MockNM nm2 = rm.registerNode("h2:1234", 80000); // app1 - RMApp app1 = rm.submitApp(GB, "app", "u1", null, "a"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("app") + .withUser("u1") + .withAcls(null) + .withQueue("a") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm2); // c2 on n1, c3 on n2 @@ -540,7 +574,15 @@ public RMNodeLabelsManager createNodeLabelManager() { RMContainerState.ALLOCATED)); // app2 - RMApp app2 = rm.submitApp(GB, "app", "u2", null, "a"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("app") + .withUser("u2") + .withAcls(null) + .withQueue("a") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm2); // c2/c3 on n1 @@ -664,7 +706,15 @@ public RMNodeLabelsManager createNodeLabelManager() { // launch an app to queue a1 (label = x), and check all container will // be allocated in h1 - RMApp app1 = rm.submitApp(GB, "app", "user", null, "a", "x"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a") + .withAmLabel("x") + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); // request a container. @@ -760,7 +810,15 @@ public RMNodeLabelsManager createNodeLabelManager() { rm.registerNode("h9:1234", 2048); // Submit app with AM container launched on default partition i.e. h1. - RMApp app = rm.submitApp(GB, "app", "user", null, "a"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a") + .withUnmanaged(false) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); MockRM.launchAndRegisterAM(app, rm, nm1); RMAppAttempt appAttempt = app.getCurrentAppAttempt(); // Add default node blacklist from default @@ -776,7 +834,15 @@ public RMNodeLabelsManager createNodeLabelManager() { Assert.assertEquals(2, blacklistUpdates.getBlacklistRemovals().size()); // Submission in label x - RMApp applabel = rm.submitApp(GB, "app", "user", null, "a", "x"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a") + .withAmLabel("x") + .build(); + RMApp applabel = MockRMAppSubmitter.submit(rm, data1); MockRM.launchAndRegisterAM(applabel, rm, nm2); RMAppAttempt appAttemptlabelx = applabel.getCurrentAppAttempt(); appAttemptlabelx.getAMBlacklistManager().addNode("h2"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSchedulingRequestUpdate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSchedulingRequestUpdate.java index ee7e013ccbd..32dcc8ad8c6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSchedulingRequestUpdate.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSchedulingRequestUpdate.java @@ -30,6 +30,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; @@ -75,11 +77,27 @@ protected RMNodeLabelsManager createNodeLabelManager() { nm2.registerNode(); // Launch app1 in queue=a1 - RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "a1"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm2); // Launch app2 in queue=b1 - RMApp app2 = rm.submitApp(8 * GB, "app", "user", null, "b1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(8 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm2); // am1 asks for 8 * 1GB container for no label am1.allocateIntraAppAntiAffinity( @@ -193,11 +211,27 @@ protected RMNodeLabelsManager createNodeLabelManager() { nm2.registerNode(); // Launch app1 in queue=a1 - RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "a1"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm2); // Launch app2 in queue=b1 - RMApp app2 = rm.submitApp(8 * GB, "app", "user", null, "b1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(8 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm2); // am1 asks for 8 * 1GB container for "x" am1.allocateIntraAppAntiAffinity("x", diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java index c2f4511f1fa..43cb8632afb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java @@ -30,6 +30,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.monitor.SchedulingMonitor; import org.apache.hadoop.yarn.server.resourcemanager.monitor.SchedulingMonitorManager; import org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy; @@ -121,7 +123,15 @@ protected RMNodeLabelsManager createNodeLabelManager() { RMNode rmNode2 = rm1.getRMContext().getRMNodes().get(nm2.getNodeId()); // launch an app to queue, AM container should be launched in nm1 - RMApp app1 = rm1.submitApp(1 * GB, "app", user1, null, queue1); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser(user1) + .withAcls(null) + .withQueue(queue1) + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); am1.allocate("*", 1 * GB, 32, new ArrayList()); @@ -144,7 +154,15 @@ protected RMNodeLabelsManager createNodeLabelManager() { // Submit app2 to queue-c and asks for a 1G container for AM - RMApp app2 = rm1.submitApp(1 * GB, "app", user2, null, queue2); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser(user2) + .withAcls(null) + .withQueue(queue2) + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1); // NM1/NM2 has available resource = 2G/4G @@ -229,7 +247,15 @@ public void testSurgicalPreemptionWithAvailableResource() RMNode rmNode2 = rm1.getRMContext().getRMNodes().get(nm2.getNodeId()); // launch an app to queue, AM container should be launched in nm1 - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "a"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); am1.allocate("*", 1 * GB, 38, new ArrayList()); @@ -252,7 +278,15 @@ public void testSurgicalPreemptionWithAvailableResource() // Submit app2 to queue-c and asks for a 4G container for AM - RMApp app2 = rm1.submitApp(4 * GB, "app", "user", null, "c"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(4 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data); FiCaSchedulerApp schedulerApp2 = cs.getApplicationAttempt( ApplicationAttemptId.newInstance(app2.getApplicationId(), 1)); @@ -333,7 +367,15 @@ public void testPriorityPreemptionWhenAllQueuesAreBelowGuaranteedCapacities() RMNode rmNode2 = rm1.getRMContext().getRMNodes().get(nm2.getNodeId()); // launch an app to queue, AM container should be launched in nm1 - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "b"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); am1.allocate("*", 1 * GB, 6, new ArrayList<>()); @@ -356,7 +398,15 @@ public void testPriorityPreemptionWhenAllQueuesAreBelowGuaranteedCapacities() am1.getApplicationAttemptId(), 3); // Submit app2 to queue-c and asks for a 1G container for AM - RMApp app2 = rm1.submitApp(18 * GB, "app", "user", null, "c"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(18 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data); FiCaSchedulerApp schedulerApp2 = cs.getApplicationAttempt( ApplicationAttemptId.newInstance(app2.getApplicationId(), 1)); @@ -442,7 +492,15 @@ public void testPriorityPreemptionRequiresMoveReservation() RMNode rmNode3 = rm1.getRMContext().getRMNodes().get(nm3.getNodeId()); // launch an app to queue, AM container should be launched in nm1 - RMApp app1 = rm1.submitApp(2 * GB, "app", "user", null, "b"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(2 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); am1.allocate("*", 2 * GB, 2, new ArrayList<>()); @@ -464,7 +522,15 @@ public void testPriorityPreemptionRequiresMoveReservation() am1.getApplicationAttemptId(), 2); // Submit app2 to queue-c and asks for a 2G container for AM, on n3 - RMApp app2 = rm1.submitApp(2 * GB, "app", "user", null, "c"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(2 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm3); FiCaSchedulerApp schedulerApp2 = cs.getApplicationAttempt( ApplicationAttemptId.newInstance(app2.getApplicationId(), 1)); @@ -567,7 +633,15 @@ public void testPriorityPreemptionOnlyTriggeredWhenDemandingQueueUnsatisfied() } // launch an app to queue, AM container should be launched in nm1 - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "b"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, mockNMs[0]); am1.allocate("*", 1 * GB, 8, new ArrayList<>()); @@ -588,7 +662,15 @@ public void testPriorityPreemptionOnlyTriggeredWhenDemandingQueueUnsatisfied() // Submit app2 to queue-c and asks for a 10G container for AM // Launch AM in NM9 - RMApp app2 = rm1.submitApp(10 * GB, "app", "user", null, "c"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(10 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, mockNMs[9]); FiCaSchedulerApp schedulerApp2 = cs.getApplicationAttempt( ApplicationAttemptId.newInstance(app2.getApplicationId(), 1)); @@ -711,7 +793,15 @@ protected RMNodeLabelsManager createNodeLabelManager() { } // launch an app to queue, AM container should be launched in nm1 - RMApp app1 = rm1.submitApp(1 * GB, "app", users[2], null, queues[2]); + MockRMAppSubmissionData data2 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser(users[2]) + .withAcls(null) + .withQueue(queues[2]) + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data2); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, mockNMs[4]); am1.allocate("*", 1 * GB, 4, new ArrayList<>()); @@ -731,7 +821,15 @@ protected RMNodeLabelsManager createNodeLabelManager() { } // Submit app2 to queue-a and asks for a 0.5G container for AM (on n0) - RMApp app2 = rm1.submitApp(512, "app", users[0], null, queues[0]); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(512, rm1) + .withName("app") + .withUser(users[0]) + .withAcls(null) + .withQueue(queues[0]) + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data1); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, mockNMs[0]); FiCaSchedulerApp schedulerApp2 = cs.getApplicationAttempt( ApplicationAttemptId.newInstance(app2.getApplicationId(), 1)); @@ -753,7 +851,15 @@ protected RMNodeLabelsManager createNodeLabelManager() { } // Submit app3 to queue-b and asks for a 0.5G container for AM (on n2) - RMApp app3 = rm1.submitApp(512, "app", users[1], null, queues[1]); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(512, rm1) + .withName("app") + .withUser(users[1]) + .withAcls(null) + .withQueue(queues[1]) + .withUnmanaged(false) + .build(); + RMApp app3 = MockRMAppSubmitter.submit(rm1, data); MockAM am3 = MockRM.launchAndRegisterAM(app3, rm1, mockNMs[2]); FiCaSchedulerApp schedulerApp3 = cs.getApplicationAttempt( ApplicationAttemptId.newInstance(app3.getApplicationId(), 1)); @@ -943,7 +1049,15 @@ protected RMNodeLabelsManager createNodeLabelManager() { } // launch an app to queue B, AM container launched in nm4 - RMApp app1 = rm1.submitApp(4096, "app", "user", null, "B"); + MockRMAppSubmissionData data2 = + MockRMAppSubmissionData.Builder.createWithDefaults(4096, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("B") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data2); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, mockNMs[4]); am1.allocate("*", 4096, NUM_NM-1, new ArrayList<>()); @@ -963,7 +1077,15 @@ protected RMNodeLabelsManager createNodeLabelManager() { } // Submit app2 to queue A and asks for a 750MB container for AM (on n0) - RMApp app2 = rm1.submitApp(1024, "app", "user", null, "A"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("A") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data1); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, mockNMs[0]); FiCaSchedulerApp schedulerApp2 = cs.getApplicationAttempt( ApplicationAttemptId.newInstance(app2.getApplicationId(), 1)); @@ -980,7 +1102,15 @@ protected RMNodeLabelsManager createNodeLabelManager() { rm1.killApp(app1.getApplicationId()); // Submit app3 to queue B and asks for a 5000MB container for AM (on n2) - RMApp app3 = rm1.submitApp(1024, "app", "user", null, "B"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("B") + .withUnmanaged(false) + .build(); + RMApp app3 = MockRMAppSubmitter.submit(rm1, data); MockAM am3 = MockRM.launchAndRegisterAM(app3, rm1, mockNMs[2]); FiCaSchedulerApp schedulerApp3 = cs.getApplicationAttempt( ApplicationAttemptId.newInstance(app3.getApplicationId(), 1)); @@ -1056,7 +1186,15 @@ public void testPreemptionForFragmentatedCluster() throws Exception { CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler(); // launch an app to queue, AM container should be launched in nm1 - RMApp app1 = rm1.submitApp(3 * GB, "app", "user", null, "a"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(3 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nms.get(0)); am1.allocate("*", 21 * GB, 4, new ArrayList()); @@ -1074,7 +1212,15 @@ public void testPreemptionForFragmentatedCluster() throws Exception { Assert.assertEquals(5, schedulerApp1.getLiveContainers().size()); // launch an app to queue, AM container should be launched in nm1 - RMApp app2 = rm1.submitApp(3 * GB, "app", "user", null, "b"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(3 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nms.get(2)); am2.allocate("*", 21 * GB, 4, new ArrayList()); @@ -1162,7 +1308,15 @@ public void testPreemptionToBalanceWithCustomTimeout() throws Exception { RMNode rmNode2 = rm1.getRMContext().getRMNodes().get(nm2.getNodeId()); // launch an app to queue, AM container should be launched in nm1 - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "a"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); am1.allocate("*", 1 * GB, 38, new ArrayList()); @@ -1185,7 +1339,15 @@ public void testPreemptionToBalanceWithCustomTimeout() throws Exception { // Submit app2 to queue-c and asks for a 4G container for AM - RMApp app2 = rm1.submitApp(4 * GB, "app", "user", null, "c"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(4 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data); FiCaSchedulerApp schedulerApp2 = cs.getApplicationAttempt( ApplicationAttemptId.newInstance(app2.getApplicationId(), 1)); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java index 720e7872540..03cbbb86cf6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java @@ -29,6 +29,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.resource.TestResourceProfiles; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState; @@ -285,7 +287,13 @@ public void testMaxLimitsOfQueueWithMultipleResources() throws Exception { rm.registerNode("127.0.0.1:1235", TestUtils.createResource(10 * GB, 4, nameToValues)); - RMApp app1 = rm.submitApp(1024, "app-1", "user1", null, "a"); + RMApp app1 = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm) + .withName("app-1") + .withUser("user1") + .withAcls(null) + .withQueue("a") + .build()); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); SchedulerNodeReport report_nm1 = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWorkflowPriorityMapping.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWorkflowPriorityMapping.java index 2a679a15e0c..13c33470cf2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWorkflowPriorityMapping.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWorkflowPriorityMapping.java @@ -28,6 +28,8 @@ import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.WorkflowPriorityMappingsManager.WorkflowPriorityMapping; @@ -97,10 +99,16 @@ public void testWorkflowPriorityMappings() throws Exception { .getWorkflowPriorityMappings()); // Maps to rule corresponding to parent queue "a" for workflow3. - mockRM.submitApp(1, "a2", true, ApplicationId.newInstance(0,1), - Priority.newInstance(0), ImmutableSet.of( - YarnConfiguration.DEFAULT_YARN_WORKFLOW_ID_TAG_PREFIX - + "workflow3")); + MockRMAppSubmitter.submit(mockRM, + MockRMAppSubmissionData.Builder.createWithDefaults(1, mockRM) + .withQueue("a2") + .withIsAppIdProvided(true) + .withApplicationId(ApplicationId.newInstance(0, 1)) + .withPriority(Priority.newInstance(0)) + .withApplicationTags(ImmutableSet.of( + YarnConfiguration.DEFAULT_YARN_WORKFLOW_ID_TAG_PREFIX + + "workflow3")) + .build()); RMApp app = mockRM.getRMContext().getRMApps().get(ApplicationId.newInstance(0,1)); assertEquals(4, app.getApplicationSubmissionContext().getPriority() @@ -108,30 +116,48 @@ public void testWorkflowPriorityMappings() throws Exception { // Does not match any rule as rule for queue + workflow does not exist. // Priority passed in the app is taken up. - mockRM.submitApp(1, "a1", true, ApplicationId.newInstance(0,2), - Priority.newInstance(6), ImmutableSet.of( - YarnConfiguration.DEFAULT_YARN_WORKFLOW_ID_TAG_PREFIX - + "workflow1")); + MockRMAppSubmitter.submit(mockRM, + MockRMAppSubmissionData.Builder.createWithDefaults(1, mockRM) + .withQueue("a1") + .withIsAppIdProvided(true) + .withApplicationId(ApplicationId.newInstance(0, 2)) + .withPriority(Priority.newInstance(6)) + .withApplicationTags(ImmutableSet.of( + YarnConfiguration.DEFAULT_YARN_WORKFLOW_ID_TAG_PREFIX + + "workflow1")) + .build()); app = mockRM.getRMContext().getRMApps().get(ApplicationId.newInstance(0,2)); assertEquals(6, app.getApplicationSubmissionContext().getPriority() .getPriority()); // Maps to rule corresponding to parent queue "a1" for workflow2. - mockRM.submitApp(1, "a1", true, ApplicationId.newInstance(0,3), - Priority.newInstance(0), ImmutableSet.of( - YarnConfiguration.DEFAULT_YARN_WORKFLOW_ID_TAG_PREFIX - + "workflow2")); + MockRMAppSubmitter.submit(mockRM, + MockRMAppSubmissionData.Builder.createWithDefaults(1, mockRM) + .withQueue("a1") + .withIsAppIdProvided(true) + .withApplicationId(ApplicationId.newInstance(0, 3)) + .withPriority(Priority.newInstance(0)) + .withApplicationTags(ImmutableSet.of( + YarnConfiguration.DEFAULT_YARN_WORKFLOW_ID_TAG_PREFIX + + "workflow2")) + .build()); app = mockRM.getRMContext().getRMApps().get(ApplicationId.newInstance(0,3)); assertEquals(3, app.getApplicationSubmissionContext().getPriority() .getPriority()); // Maps to rule corresponding to parent queue "b" for workflow1. - mockRM.submitApp(1, "b3", true, ApplicationId.newInstance(0,4), - Priority.newInstance(0), ImmutableSet.of( - YarnConfiguration.DEFAULT_YARN_WORKFLOW_ID_TAG_PREFIX - + "workflow1")); + MockRMAppSubmitter.submit(mockRM, + MockRMAppSubmissionData.Builder.createWithDefaults(1, mockRM) + .withQueue("b3") + .withIsAppIdProvided(true) + .withApplicationId(ApplicationId.newInstance(0, 4)) + .withPriority(Priority.newInstance(0)) + .withApplicationTags(ImmutableSet.of( + YarnConfiguration.DEFAULT_YARN_WORKFLOW_ID_TAG_PREFIX + + "workflow1")) + .build()); app = mockRM.getRMContext().getRMApps().get(ApplicationId.newInstance(0,4)); assertEquals(2, app.getApplicationSubmissionContext().getPriority() .getPriority()); @@ -140,10 +166,17 @@ public void testWorkflowPriorityMappings() throws Exception { conf.setBoolean(CapacitySchedulerConfiguration .ENABLE_WORKFLOW_PRIORITY_MAPPINGS_OVERRIDE, false); cs.reinitialize(conf, mockRM.getRMContext()); - mockRM.submitApp(1, "a2", true, ApplicationId.newInstance(0,5), - Priority.newInstance(0), ImmutableSet.of( - YarnConfiguration.DEFAULT_YARN_WORKFLOW_ID_TAG_PREFIX - + "workflow3")); + + MockRMAppSubmitter.submit(mockRM, + MockRMAppSubmissionData.Builder.createWithDefaults(1, mockRM) + .withQueue("a2") + .withIsAppIdProvided(true) + .withApplicationId(ApplicationId.newInstance(0, 5)) + .withPriority(Priority.newInstance(0)) + .withApplicationTags(ImmutableSet.of( + YarnConfiguration.DEFAULT_YARN_WORKFLOW_ID_TAG_PREFIX + + "workflow3")) + .build()); app = mockRM.getRMContext().getRMApps().get(ApplicationId.newInstance(0,5)); assertEquals(0, app.getApplicationSubmissionContext().getPriority() .getPriority()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java index 22b311db4dd..b1e4cfb5f92 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java @@ -44,6 +44,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; import org.apache.hadoop.yarn.server.resourcemanager.RMSecretManagerService; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager; @@ -115,7 +117,7 @@ public void testExcessReservationThanNodeManagerCapacity() throws Exception { } Assert.assertEquals(2, rm.getRMContext().getRMNodes().size()); // Submit an application - RMApp app1 = rm.submitApp(128); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(128, rm); // kick the scheduling nm1.nodeHeartbeat(true); @@ -167,7 +169,7 @@ public void testContainerTokenGeneratedOnPullRequest() throws Exception { MockRM rm1 = new MockRM(conf); rm1.start(); MockNM nm1 = rm1.registerNode("127.0.0.1:1234", 8000); - RMApp app1 = rm1.submitApp(200); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // request a container. am1.allocate("127.0.0.1", 1024, 1, new ArrayList()); @@ -196,7 +198,7 @@ public void testNormalContainerAllocationWhenDNSUnavailable() throws Exception{ MockRM rm1 = new MockRM(conf); rm1.start(); MockNM nm1 = rm1.registerNode("unknownhost:1234", 8000); - RMApp app1 = rm1.submitApp(200); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // request a container. @@ -264,7 +266,10 @@ public void testLogAggregationContextPassedIntoContainerToken() private LogAggregationContext getLogAggregationContextFromContainerToken( MockRM rm1, MockNM nm1, LogAggregationContext logAggregationContext) throws Exception { - RMApp app2 = rm1.submitApp(200, logAggregationContext); + RMApp app2 = MockRMAppSubmitter.submit(rm1, + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withLogAggregationContext(logAggregationContext) + .build()); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1); nm1.nodeHeartbeat(true); // request a container. @@ -332,7 +337,7 @@ protected RMSecretManagerService createRMSecretManagerService() { RMApp app1; try { SecurityUtilTestHelper.setTokenServiceUseIp(true); - app1 = rm1.submitApp(200); + app1 = MockRMAppSubmitter.submitWithMemory(200, rm1); RMAppAttempt attempt = app1.getCurrentAppAttempt(); nm1.nodeHeartbeat(true); @@ -370,11 +375,27 @@ public void testExcessReservationWillBeUnreserved() throws Exception { MockNM nm2 = rm1.registerNode("h2:1234", 8 * GB); // launch an app to queue, AM container should be launched in nm1 - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // launch another app to queue, AM container should be launched in nm1 - RMApp app2 = rm1.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1); am1.allocate("*", 4 * GB, 1, new ArrayList()); @@ -453,11 +474,27 @@ public void testAllocationForReservedContainer() throws Exception { MockNM nm2 = rm1.registerNode("h2:1234", 8 * GB); // launch an app to queue, AM container should be launched in nm1 - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // launch another app to queue, AM container should be launched in nm1 - RMApp app2 = rm1.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1); am1.allocate("*", 4 * GB, 1, new ArrayList()); @@ -549,11 +586,27 @@ public void testReservedContainerMetricsOnDecommisionedNode() throws Exception { MockNM nm2 = rm1.registerNode("h2:1234", 8 * GB); // launch an app to queue, AM container should be launched in nm1 - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // launch another app to queue, AM container should be launched in nm1 - RMApp app2 = rm1.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1); am1.allocate("*", 4 * GB, 1, new ArrayList()); @@ -621,7 +674,15 @@ public void testAssignMultipleOffswitchContainers() throws Exception { MockNM nm1 = rm1.registerNode("h1:1234", 80 * GB); // launch an app to queue, AM container should be launched in nm1 - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); am1.allocate("*", 1 * GB, 5, new ArrayList()); @@ -670,12 +731,28 @@ public void testContinuousReservationLookingWhenUsedEqualsMax() throws Exception MockNM nm2 = rm1.registerNode("h2:1234", 90 * GB); // launch an app to queue A, AM container should be launched in nm1 - RMApp app1 = rm1.submitApp(2 * GB, "app", "user", null, "a"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(2 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // launch 2nd app to queue B, AM container should be launched in nm1 // Now usage of nm1 is 3G (2G + 1G) - RMApp app2 = rm1.submitApp(1 * GB, "app", "user", null, "b"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1); am1.allocate("*", 4 * GB, 2, null); @@ -722,11 +799,27 @@ public void testPendingResourcesConsideringUserLimit() throws Exception { MockNM nm2 = rm1.registerNode("h2:1234", 8 * GB); // launch an app to queue default, AM container should be launched in nm1 - RMApp app1 = rm1.submitApp(2 * GB, "app", "u1", null, "default"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(2 * GB, rm1) + .withName("app") + .withUser("u1") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // launch 2nd app to queue default, AM container should be launched in nm1 - RMApp app2 = rm1.submitApp(4 * GB, "app", "u2", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(4 * GB, rm1) + .withName("app") + .withUser("u2") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1); // am1 asks 1 * 3G container @@ -813,15 +906,39 @@ public void testQueuePriorityOrdering() throws Exception { MockNM nm1 = rm1.registerNode("h1:1234", 100 * GB); // launch an app to queue A, AM container should be launched in nm1 - RMApp app1 = rm1.submitApp(2 * GB, "app", "user", null, "a"); + MockRMAppSubmissionData data2 = + MockRMAppSubmissionData.Builder.createWithDefaults(2 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data2); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // launch an app to queue B, AM container should be launched in nm1 - RMApp app2 = rm1.submitApp(2 * GB, "app", "user", null, "b"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(2 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data1); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1); // launch an app to queue C, AM container should be launched in nm1 - RMApp app3 = rm1.submitApp(2 * GB, "app", "user", null, "c"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(2 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c") + .withUnmanaged(false) + .build(); + RMApp app3 = MockRMAppSubmitter.submit(rm1, data); MockAM am3 = MockRM.launchAndRegisterAM(app3, rm1, nm1); // Each application asks 10 * 5GB containers @@ -918,11 +1035,27 @@ public void testUserLimitAllocationMultipleContainers() throws Exception { MockNM nm1 = rm1.registerNode("h1:1234", 1000 * GB); // launch app from 1st user to queue C, AM container should be launched in nm1 - RMApp app1 = rm1.submitApp(2 * GB, "app", "user1", null, "c"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(2 * GB, rm1) + .withName("app") + .withUser("user1") + .withAcls(null) + .withQueue("c") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // launch app from 2nd user to queue C, AM container should be launched in nm1 - RMApp app2 = rm1.submitApp(2 * GB, "app", "user2", null, "c"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(2 * GB, rm1) + .withName("app") + .withUser("user2") + .withAcls(null) + .withQueue("c") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1); // Each application asks 1000 * 5GB containers @@ -965,15 +1098,47 @@ public void testActiveUsersWithOnlyPendingApps() throws Exception { rm1.start(); MockNM nm1 = rm1.registerNode("h1:1234", 8 * GB); - RMApp app1 = rm1.submitApp(1 * GB, "app", "u1", null, "default"); + MockRMAppSubmissionData data3 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("u1") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data3); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); - RMApp app2 = rm1.submitApp(1 * GB, "app", "u2", null, "default"); + MockRMAppSubmissionData data2 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("u2") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data2); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1); - RMApp app3 = rm1.submitApp(1 * GB, "app", "u3", null, "default"); - - RMApp app4 = rm1.submitApp(1 * GB, "app", "u4", null, "default"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("u3") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app3 = MockRMAppSubmitter.submit(rm1, data1); + + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("u4") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app4 = MockRMAppSubmitter.submit(rm1, data); // Each application asks 50 * 1GB containers am1.allocate("*", 1 * GB, 50, null); @@ -1031,11 +1196,27 @@ public void testUnreserveWhenClusterResourceHasEmptyResourceType() MockNM nm2 = rm1.registerNode("h2:1234", 8 * GB); // launch an app to queue "a", AM container should be launched on nm1 - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "a"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // launch another app to queue "b", AM container should be launched on nm1 - RMApp app2 = rm1.submitApp(1 * GB, "app", "user", null, "b"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data); MockRM.launchAndRegisterAM(app2, rm1, nm1); am1.allocate("*", 7 * GB, 2, new ArrayList()); @@ -1112,11 +1293,27 @@ public void testAllocationCannotBeBlockedWhenFormerQueueReachedItsLimit() MockNM nm2 = rm1.registerNode("h2:1234", 10 * GB); // launch an app to queue "c1", AM container should be launched on nm1 - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "c1"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // launch another app to queue "b", AM container should be launched on nm1 - RMApp app2 = rm1.submitApp(1 * GB, "app", "user", null, "b"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1); am1.allocate("*", 2 * GB, 1, new ArrayList()); @@ -1156,7 +1353,15 @@ public void testContainerRejectionWhenAskBeyondDynamicMax() // submit an app beyond queue max leads to failure. boolean submitFailed = false; MockNM nm1 = rm1.registerNode("h1:1234", 2 * GB, 1); - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "a"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); try { am1.allocate("*", 5 * GB, 1, null); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerResizing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerResizing.java index f9bc7c4be32..d829389edd5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerResizing.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerResizing.java @@ -42,6 +42,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; @@ -125,7 +127,15 @@ public RMNodeLabelsManager createNodeLabelManager() { MockNM nm1 = rm1.registerNode("h1:1234", 20 * GB); // app1 -> a1 - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); ContainerId containerId1 = @@ -183,7 +193,15 @@ protected Dispatcher createDispatcher() { MockNM nm1 = rm1.registerNode("h1:1234", 20 * GB); // app1 -> a1 - RMApp app1 = rm1.submitApp(3 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(3 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); FiCaSchedulerApp app = TestUtils.getFiCaSchedulerApp( rm1, app1.getApplicationId()); @@ -248,7 +266,15 @@ public RMNodeLabelsManager createNodeLabelManager() { MockNM nm2 = rm1.registerNode("h2:1234", 8 * GB); // app1 -> a1 - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); FiCaSchedulerApp app = TestUtils.getFiCaSchedulerApp( @@ -349,7 +375,15 @@ public RMNodeLabelsManager createNodeLabelManager() { MockNM nm1 = rm1.registerNode("h1:1234", 8 * GB); // app1 -> a1 - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); FiCaSchedulerApp app = TestUtils.getFiCaSchedulerApp( @@ -424,7 +458,15 @@ public RMNodeLabelsManager createNodeLabelManager() { MockNM nm2 = rm1.registerNode("h2:1234", 8 * GB); // app1 -> a1 - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); FiCaSchedulerApp app = TestUtils.getFiCaSchedulerApp( @@ -538,7 +580,15 @@ protected Dispatcher createDispatcher() { MockNM nm2 = rm1.registerNode("h2:1234", 8 * GB); // app1 -> a1 - RMApp app1 = rm1.submitApp(2 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(2 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); FiCaSchedulerApp app = TestUtils.getFiCaSchedulerApp( @@ -655,7 +705,15 @@ protected Dispatcher createDispatcher() { MockNM nm2 = rm1.registerNode("h2:1234", 8 * GB); // app1 -> a1 - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); FiCaSchedulerApp app = TestUtils.getFiCaSchedulerApp( @@ -759,7 +817,15 @@ public RMNodeLabelsManager createNodeLabelManager() { MockNM nm2 = rm1.registerNode("h2:1234", 8 * GB); // app1 -> a1 - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); FiCaSchedulerApp app = TestUtils.getFiCaSchedulerApp( @@ -883,7 +949,15 @@ public RMNodeLabelsManager createNodeLabelManager() { MockNM nm1 = rm1.registerNode("h1:1234", 10 * GB); // app1 -> a1 - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); FiCaSchedulerApp app = TestUtils.getFiCaSchedulerApp( @@ -964,7 +1038,15 @@ public ResourceScheduler createScheduler() { // register a node MockNM nm = rm.registerNode("h1:1234", 20 * GB); // submit an application -> app1 - RMApp app1 = rm.submitApp(3 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(3 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm); // making sure resource is allocated checkUsedResource(rm, "default", 3 * GB, null); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestIncreaseAllocationExpirer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestIncreaseAllocationExpirer.java index a800bef95ea..4ebfc0bbb50 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestIncreaseAllocationExpirer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestIncreaseAllocationExpirer.java @@ -33,6 +33,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; @@ -82,7 +84,15 @@ public void testContainerIsRemovedFromAllocationExpirer() rm1.start(); // Submit an application MockNM nm1 = rm1.registerNode("127.0.0.1:1234", 20 * GB); - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // Report AM container status RUNNING to remove it from expirer nm1.nodeHeartbeat( @@ -166,7 +176,15 @@ protected Dispatcher createDispatcher() { }; rm1.start(); MockNM nm1 = rm1.registerNode("127.0.0.1:1234", 20 * GB); - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); nm1.nodeHeartbeat( app1.getCurrentAppAttempt() @@ -242,7 +260,15 @@ public void testConsecutiveContainerIncreaseAllocationExpiration() rm1.start(); // Submit an application MockNM nm1 = rm1.registerNode("127.0.0.1:1234", 20 * GB); - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); nm1.nodeHeartbeat( app1.getCurrentAppAttempt() @@ -359,7 +385,15 @@ public void testDecreaseAfterIncreaseWithAllocationExpiration() rm1.start(); // Submit an application MockNM nm1 = rm1.registerNode("127.0.0.1:1234", 20 * GB); - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); nm1.nodeHeartbeat( app1.getCurrentAppAttempt() diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java index 737db5b0d65..ffdca6996ef 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java @@ -37,6 +37,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; @@ -178,7 +180,15 @@ public RMNodeLabelsManager createNodeLabelManager() { // launch an app to queue a1 (label = x), and check all container will // be allocated in h1 - RMApp app1 = rm1.submitApp(200, "app", "user", null, "a1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // A has only 10% of x, so it can only allocate one container in label=empty @@ -268,7 +278,15 @@ public RMNodeLabelsManager createNodeLabelManager() { // launch an app to queue a1 (label = x), and check all container will // be allocated in h1 - RMApp app1 = rm1.submitApp(1024, "app", "user", null, "a1"); + MockRMAppSubmissionData data2 = + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data2); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // request a container (label = y). can be allocated on nm2 @@ -282,7 +300,15 @@ public RMNodeLabelsManager createNodeLabelManager() { // launch an app to queue b1 (label = y), and check all container will // be allocated in h5 - RMApp app2 = rm1.submitApp(1024, "app", "user", null, "b1"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data1); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm5); // request a container for AM, will succeed @@ -296,7 +322,15 @@ public RMNodeLabelsManager createNodeLabelManager() { RMContainerState.ALLOCATED)); // launch an app to queue b2 - RMApp app3 = rm1.submitApp(1024, "app", "user", null, "b2"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b2") + .withUnmanaged(false) + .build(); + RMApp app3 = MockRMAppSubmitter.submit(rm1, data); MockAM am3 = MockRM.launchAndRegisterAM(app3, rm1, nm5); // request a container. try to allocate on nm1 (label = x) and nm3 (label = @@ -347,7 +381,15 @@ public RMNodeLabelsManager createNodeLabelManager() { // launch an app to queue a1 (label = x), and check all container will // be allocated in h1 - RMApp app1 = rm1.submitApp(200, "app", "user", null, "a1"); + MockRMAppSubmissionData data2 = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data2); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm3); // request a container. @@ -363,7 +405,15 @@ public RMNodeLabelsManager createNodeLabelManager() { // launch an app to queue b1 (label = y), and check all container will // be allocated in h2 - RMApp app2 = rm1.submitApp(200, "app", "user", null, "b1"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data1); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm3); // request a container. @@ -378,7 +428,15 @@ public RMNodeLabelsManager createNodeLabelManager() { // launch an app to queue c1 (label = ""), and check all container will // be allocated in h3 - RMApp app3 = rm1.submitApp(200, "app", "user", null, "c1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c1") + .withUnmanaged(false) + .build(); + RMApp app3 = MockRMAppSubmitter.submit(rm1, data); MockAM am3 = MockRM.launchAndRegisterAM(app3, rm1, nm3); // request a container. @@ -423,7 +481,15 @@ public RMNodeLabelsManager createNodeLabelManager() { // launch an app to queue a1 (label = x), and check all container will // be allocated in h1 - RMApp app1 = rm1.submitApp(200, "app", "user", null, "a1"); + MockRMAppSubmissionData data2 = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data2); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // request a container. @@ -439,7 +505,15 @@ public RMNodeLabelsManager createNodeLabelManager() { // launch an app to queue b1 (label = y), and check all container will // be allocated in h2 - RMApp app2 = rm1.submitApp(200, "app", "user", null, "b1"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data1); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm2); // request a container. @@ -454,7 +528,15 @@ public RMNodeLabelsManager createNodeLabelManager() { // launch an app to queue c1 (label = ""), and check all container will // be allocated in h3 - RMApp app3 = rm1.submitApp(200, "app", "user", null, "c1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c1") + .withUnmanaged(false) + .build(); + RMApp app3 = MockRMAppSubmitter.submit(rm1, data); MockAM am3 = MockRM.launchAndRegisterAM(app3, rm1, nm3); // request a container. @@ -503,7 +585,15 @@ public RMNodeLabelsManager createNodeLabelManager() { // launch an app to queue a1 (label = x), and check all container will // be allocated in h1 - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "a1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // request a container. @@ -569,10 +659,26 @@ public void testRMContainerLeakInLeafQueue() throws Exception { // launch an app to queue a1 (label = x), and check all container will // be allocated in h1 - RMApp app1 = rm1.submitApp(1 * GB, "app1", "user", null, "a1"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app1") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); - RMApp app2 = rm1.submitApp(1 * GB, "app2", "user", null, "a1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app2") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data); MockRM.launchAndRegisterAM(app2, rm1, nm1); // request a container. @@ -654,7 +760,15 @@ public RMNodeLabelsManager createNodeLabelManager() { MockNM nm2 = rm1.registerNode("h2:1234", 40 * GB); // label = y // launch an app to queue b1 (label = y), AM container should be launched in // nm2 - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "b1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm2); // Creating request set when request before ANY is not having label and any // is having label @@ -752,11 +866,27 @@ public RMNodeLabelsManager createNodeLabelManager() { MockNM nm2 = rm1.registerNode("h2:1234", 100 * GB); // label = // launch an app to queue b1 (label = y), AM container should be launched in nm2 - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "b1"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm2); // launch another app to queue b1 (label = y), AM container should be launched in nm2 - RMApp app2 = rm1.submitApp(1 * GB, "app", "user", null, "b1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm2); // request container and nm1 do heartbeat (nm2 has label=y), note that app1 @@ -830,7 +960,15 @@ public RMNodeLabelsManager createNodeLabelManager() { MockNM nm2 = rm1.registerNode("h2:1234", 100 * GB); // label = // launch an app to queue b1 (label = y), AM container should be launched in nm3 - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "b1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm2); // request containers from am2, priority=1 asks for "" and priority=2 asks @@ -881,7 +1019,15 @@ public RMNodeLabelsManager createNodeLabelManager() { ContainerId nextContainerId; // launch an app to queue b1 (label = y), AM container should be launched in nm3 - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "b1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm2); // request containers from am2, priority=1 asks for "" * 6 (id from 4 to 9), @@ -999,27 +1145,75 @@ public RMNodeLabelsManager createNodeLabelManager() { MockNM nm2 = rm1.registerNode("h2:1234", 100 * GB); // label = // app1 -> a1 - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "a1"); + MockRMAppSubmissionData data5 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data5); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // app2 -> a2 - RMApp app2 = rm1.submitApp(1 * GB, "app", "user", null, "a2"); + MockRMAppSubmissionData data4 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a2") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data4); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm2); // app3 -> b1 - RMApp app3 = rm1.submitApp(1 * GB, "app", "user", null, "b1"); + MockRMAppSubmissionData data3 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + RMApp app3 = MockRMAppSubmitter.submit(rm1, data3); MockAM am3 = MockRM.launchAndRegisterAM(app3, rm1, nm1); // app4 -> b2 - RMApp app4 = rm1.submitApp(1 * GB, "app", "user", null, "b2"); + MockRMAppSubmissionData data2 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b2") + .withUnmanaged(false) + .build(); + RMApp app4 = MockRMAppSubmitter.submit(rm1, data2); MockAM am4 = MockRM.launchAndRegisterAM(app4, rm1, nm2); // app5 -> c1 - RMApp app5 = rm1.submitApp(1 * GB, "app", "user", null, "c1"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c1") + .withUnmanaged(false) + .build(); + RMApp app5 = MockRMAppSubmitter.submit(rm1, data1); MockAM am5 = MockRM.launchAndRegisterAM(app5, rm1, nm1); // app6 -> b2 - RMApp app6 = rm1.submitApp(1 * GB, "app", "user", null, "c2"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c2") + .withUnmanaged(false) + .build(); + RMApp app6 = MockRMAppSubmitter.submit(rm1, data); MockAM am6 = MockRM.launchAndRegisterAM(app6, rm1, nm2); // Each application request 5 * 1GB container @@ -1108,7 +1302,15 @@ public RMNodeLabelsManager createNodeLabelManager() { MockNM nm2 = rm1.registerNode("h2:1234", 10 * GB); // label = // app1 -> b - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "b"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm2); // Each application request 50 * 1GB container @@ -1172,7 +1374,15 @@ public RMNodeLabelsManager createNodeLabelManager() { MockNM nm1 = rm1.registerNode(nodeIdStr, 8 * GB); // label = x // launch an app to queue b1 (label = y), AM container should be launched in nm3 - RMApp app = rm1.submitApp(1 * GB, "app", "user", null, "b1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm1, data); CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler(); RMNode rmNode1 = rm1.getRMContext().getRMNodes().get(nm1.getNodeId()); @@ -1259,7 +1469,15 @@ public RMNodeLabelsManager createNodeLabelManager() { MockNM nm2 = rm1.registerNode("h2:1234", 10 * GB); // label = // app1 -> a - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "a"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm2); // app1 asks for 10 partition= containers @@ -1447,7 +1665,15 @@ public RMNodeLabelsManager createNodeLabelManager() { checkQueueUsedCapacity("root", cs, "", 0f, 0f); // app1 -> a1 - RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "a1"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm2); // app1 asks for 1 partition= containers @@ -1498,7 +1724,15 @@ public RMNodeLabelsManager createNodeLabelManager() { // submit an app to a2, uses 1 NON_PARTITIONED container and 1 PARTITIONED // container // app2 -> a2 - RMApp app2 = rm.submitApp(1 * GB, "app", "user", null, "a2"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a2") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm2); // app1 asks for 1 partition= containers @@ -1655,19 +1889,51 @@ public RMNodeLabelsManager createNodeLabelManager() { MockNM nm2 = rm.registerNode("h2:1234", 10 * GB); // label = // app1 -> a - RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "a"); + MockRMAppSubmissionData data3 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data3); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm2); // app2 -> b - RMApp app2 = rm.submitApp(1 * GB, "app", "user", null, "b"); + MockRMAppSubmissionData data2 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm, data2); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm2); // app3 -> c - RMApp app3 = rm.submitApp(1 * GB, "app", "user", null, "c"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c") + .withUnmanaged(false) + .build(); + RMApp app3 = MockRMAppSubmitter.submit(rm, data1); MockAM am3 = MockRM.launchAndRegisterAM(app3, rm, nm2); // app4 -> d - RMApp app4 = rm.submitApp(1 * GB, "app", "user", null, "d"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("d") + .withUnmanaged(false) + .build(); + RMApp app4 = MockRMAppSubmitter.submit(rm, data); MockAM am4 = MockRM.launchAndRegisterAM(app4, rm, nm2); // Test case 1 @@ -1817,11 +2083,27 @@ public RMNodeLabelsManager createNodeLabelManager() { MockNM nm1 = rm.registerNode("h1:1234", 10 * GB); // label = x // app1 -> a - RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "a", "x"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a") + .withAmLabel("x") + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); // app2 -> b - RMApp app2 = rm.submitApp(1 * GB, "app", "user", null, "b", "x"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b") + .withAmLabel("x") + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm1); // Both a/b has used_capacity(x) = 0, when doing exclusive allocation, a @@ -1909,7 +2191,15 @@ public RMNodeLabelsManager createNodeLabelManager() { nm1.registerNode(); // Launch app1 in a1, resource usage is 1GB (am) + 4GB * 2 = 9GB - RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "a1", "x"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withAmLabel("x") + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); am1.allocate("*", 4 * GB, 2, new ArrayList(), "x"); doNMHeartbeat(rm, nm1.getNodeId(), 10); @@ -1917,7 +2207,15 @@ public RMNodeLabelsManager createNodeLabelManager() { cs.getApplicationAttempt(am1.getApplicationAttemptId())); // Try to launch app2 in a2, asked 2GB, should success - RMApp app2 = rm.submitApp(2 * GB, "app", "user", null, "a2", "x"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(2 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a2") + .withAmLabel("x") + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm1); // am2 asks more resources, cannot success because current used = 9G (app1) @@ -1998,7 +2296,15 @@ public RMNodeLabelsManager createNodeLabelManager() { assertEquals(0 * GB, leafQueueB.getMetrics().getAllocatedMB()); // app1 -> a - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "a", "x"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a") + .withAmLabel("x") + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // app1 asks for 5 partition=x containers @@ -2103,7 +2409,15 @@ public RMNodeLabelsManager createNodeLabelManager() { MockNM nm1 = rm1.registerNode("h1:1234", 10 * GB); // label = x MockNM nm2 = rm1.registerNode("h2:1234", 10 * GB); // label = // app1 -> a - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "a"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm2); // app1 asks for 3 partition= containers @@ -2212,7 +2526,14 @@ public RMNodeLabelsManager createNodeLabelManager() { assertEquals(0 * GB, leafQueueA.getMetrics().getAllocatedMB()); // app1 -> a - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "a", "x"); + RMApp app1 = MockRMAppSubmitter.submit(rm1, + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a") + .withAmLabel("x") + .build()); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // app1 asks for 5 partition=x containers @@ -2245,7 +2566,14 @@ public RMNodeLabelsManager createNodeLabelManager() { assertEquals(0 * GB, leafQueueA.getMetrics().getAllocatedMB()); // app2 -> a - RMApp app2 = rm1.submitApp(1 * GB, "app", "user", null, "a", ""); + RMApp app2 = MockRMAppSubmitter.submit(rm1, + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a") + .withAmLabel("") + .build()); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm2); // app2 asks for 5 partition= containers @@ -2370,7 +2698,15 @@ public RMNodeLabelsManager createNodeLabelManager() { rm1.start(); MockNM nm1 = rm1.registerNode("h1:1234", 20 * GB); // label = x // app1 -> a1 - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "a1", "x"); + MockRMAppSubmissionData data3 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withAmLabel("x") + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data3); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // app1 asks for 6 partition=x containers @@ -2390,7 +2726,15 @@ public RMNodeLabelsManager createNodeLabelManager() { // Try to launch app2 in a2, asked 2GB, should success // app2 -> a2 - RMApp app2 = rm1.submitApp(1 * GB, "app", "user", null, "a2", "x"); + MockRMAppSubmissionData data2 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a2") + .withAmLabel("x") + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data2); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1); // app2 asks for 4 partition=x containers @@ -2414,7 +2758,15 @@ public RMNodeLabelsManager createNodeLabelManager() { // Try to launch app3 in a2, asked 6GB, should fail // app3 -> a2 - RMApp app3 = rm1.submitApp(1 * GB, "app", "user", null, "a2", "x"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a2") + .withAmLabel("x") + .build(); + RMApp app3 = MockRMAppSubmitter.submit(rm1, data1); MockAM am3 = MockRM.launchAndRegisterAM(app3, rm1, nm1); am3.allocate("*", 1 * GB, 6, new ArrayList(), "x"); @@ -2434,7 +2786,15 @@ public RMNodeLabelsManager createNodeLabelManager() { rm1.waitForAppRemovedFromScheduler(app1.getApplicationId()); // app4 -> a1, try to allocate more than 6GB resource, should fail - RMApp app4 = rm1.submitApp(1 * GB, "app", "user", null, "a1", "x"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withAmLabel("x") + .build(); + RMApp app4 = MockRMAppSubmitter.submit(rm1, data); MockAM am4 = MockRM.launchAndRegisterAM(app4, rm1, nm1); // app3 asks for 7 partition=x containers diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueState.java index c9980ceae44..44d8ce1927b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueState.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueState.java @@ -33,6 +33,8 @@ import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; @@ -226,7 +228,15 @@ public void testRecoverDrainingStateAfterRMRestart() throws Exception { MockNM nm = rm.registerNode("h1:1234", 204800); // submit an app, AM is running on nm1 - RMApp app = rm.submitApp(1024, "appname", "appuser", null, Q2); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm) + .withName("appname") + .withUser("appuser") + .withAcls(null) + .withQueue(Q2) + .withUnmanaged(false) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); MockRM.launchAM(app, rm, nm); rm.waitForState(app.getApplicationId(), RMAppState.ACCEPTED); // update queue state to STOPPED diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestSchedulingRequestContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestSchedulingRequestContainerAllocation.java index 2c1476e4b65..953b632ab11 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestSchedulingRequestContainerAllocation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestSchedulingRequestContainerAllocation.java @@ -24,8 +24,11 @@ import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; import org.apache.hadoop.yarn.api.records.ExecutionType; import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest; +import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.SchedulingRequest; import org.apache.hadoop.yarn.api.resource.PlacementConstraint; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.TargetApplicationsNamespace; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; @@ -56,6 +59,7 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.Collection; import java.util.HashMap; import java.util.List; @@ -106,6 +110,21 @@ public void setUp() throws Exception { mgr.init(conf); } + private RMApp submitApp(MockRM rm, int memory, Set appTags) throws Exception { + Resource resource = Resource.newInstance(memory, 0); + ResourceRequest amResourceRequest = ResourceRequest.newInstance( + Priority.newInstance(0), ResourceRequest.ANY, resource, 1); + List amResourceRequests = + Collections.singletonList(amResourceRequest); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(resource, rm) + .withAmLabel(null) + .withAmResourceRequests(amResourceRequests) + .withApplicationTags(appTags) + .build(); + return MockRMAppSubmitter.submit(rm, data); + } + @Test(timeout = 30000L) public void testIntraAppAntiAffinity() throws Exception { Configuration csConf = TestUtils.getConfigurationWithMultipleQueues(conf); @@ -130,7 +149,15 @@ public RMNodeLabelsManager createNodeLabelManager() { } // app1 -> c - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "c"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nms[0]); // app1 asks for 10 anti-affinity containers for the same app. It should @@ -196,7 +223,15 @@ public RMNodeLabelsManager createNodeLabelManager() { } // app1 -> c - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "c"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nms[0]); // app1 asks for 2 anti-affinity containers for the same app. @@ -266,7 +301,15 @@ public RMNodeLabelsManager createNodeLabelManager() { } // app1 -> c - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "c"); + MockRMAppSubmissionData data2 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data2); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nms[0]); // app1 asks for 3 anti-affinity containers for the same app. It should @@ -287,7 +330,15 @@ public RMNodeLabelsManager createNodeLabelManager() { + rmNodes[2].getAllocationTagsWithCount().get("mapper")); // app2 -> c - RMApp app2 = rm1.submitApp(1 * GB, "app", "user", null, "c"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data1); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nms[0]); // App2 asks for 3 containers that anti-affinity with any mapper, @@ -322,7 +373,15 @@ public RMNodeLabelsManager createNodeLabelManager() { })); // app3 -> c - RMApp app3 = rm1.submitApp(1 * GB, "app", "user", null, "c"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c") + .withUnmanaged(false) + .build(); + RMApp app3 = MockRMAppSubmitter.submit(rm1, data); MockAM am3 = MockRM.launchAndRegisterAM(app3, rm1, nms[0]); // App3 asks for 3 containers that anti-affinity with any mapper. @@ -368,7 +427,15 @@ public RMNodeLabelsManager createNodeLabelManager() { } // app1 -> c - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "c"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nms[0]); // app1 asks for 2 anti-affinity containers for the same app. @@ -410,7 +477,15 @@ public RMNodeLabelsManager createNodeLabelManager() { } // app1 -> c - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "c"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nms[0]); CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler(); @@ -528,7 +603,7 @@ public void testInterAppCompositeConstraints() throws Exception { MockNM nm4 = rm.registerNode("192.168.0.4:1234", 100*GB, 100); MockNM nm5 = rm.registerNode("192.168.0.5:1234", 100*GB, 100); - RMApp app1 = rm.submitApp(1*GB, ImmutableSet.of("hbase")); + RMApp app1 = submitApp(rm, 1*GB, ImmutableSet.of("hbase")); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); // App1 (hbase) @@ -567,7 +642,7 @@ public void testInterAppCompositeConstraints() throws Exception { // App2 (web-server) // Web server instance has 2 instance and non of them can be co-allocated // with hbase-master. - RMApp app2 = rm.submitApp(1*GB, ImmutableSet.of("web-server")); + RMApp app2 = submitApp(rm, 1*GB, ImmutableSet.of("web-server")); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm2); // App2 (web-server) @@ -604,7 +679,7 @@ public void testInterAppCompositeConstraints() throws Exception { // App3 has multiple instances that must be co-allocated // with app2 server instance, and each node cannot have more than // 3 instances. - RMApp app3 = rm.submitApp(1*GB, ImmutableSet.of("ws-servants")); + RMApp app3 = submitApp(rm, 1*GB, ImmutableSet.of("ws-servants")); MockAM am3 = MockRM.launchAndRegisterAM(app3, rm, nm3); @@ -656,7 +731,7 @@ public void testMultiAllocationTagsConstraints() throws Exception { MockNM nm4 = rm.registerNode("192.168.0.4:1234", 10*GB, 10); MockNM nm5 = rm.registerNode("192.168.0.5:1234", 10*GB, 10); - RMApp app1 = rm.submitApp(1*GB, ImmutableSet.of("server1")); + RMApp app1 = submitApp(rm, 1*GB, ImmutableSet.of("server1")); // Allocate AM container on nm1 doNodeHeartbeat(nm1); RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); @@ -683,7 +758,7 @@ public void testMultiAllocationTagsConstraints() throws Exception { // App1 uses ports: 6000 String[] server2Ports = new String[] {"port_6000"}; - RMApp app2 = rm.submitApp(1*GB, ImmutableSet.of("server2")); + RMApp app2 = submitApp(rm, 1*GB, ImmutableSet.of("server2")); // Allocate AM container on nm1 doNodeHeartbeat(nm2); RMAppAttempt app2attempt1 = app2.getCurrentAppAttempt(); @@ -739,7 +814,7 @@ public void testInterAppConstraintsWithNamespaces() throws Exception { // App1 ~ app5 tag "former5" // App6 ~ app10 tag "latter5" String applicationTag = i<5 ? "former5" : "latter5"; - RMApp app = rm.submitApp(1*GB, ImmutableSet.of(applicationTag)); + RMApp app = submitApp(rm, 1*GB, ImmutableSet.of(applicationTag)); // Allocate AM container on nm1 doNodeHeartbeat(nm1, nm2, nm3, nm4, nm5); RMAppAttempt attempt = app.getCurrentAppAttempt(); @@ -766,7 +841,7 @@ public void testInterAppConstraintsWithNamespaces() throws Exception { // *** app-id // Submit another app, use app-id constraint against app5 - RMApp app1 = rm.submitApp(1*GB, ImmutableSet.of("xyz")); + RMApp app1 = submitApp(rm, 1*GB, ImmutableSet.of("xyz")); // Allocate AM container on nm1 doNodeHeartbeat(nm1); RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); @@ -796,7 +871,7 @@ public void testInterAppConstraintsWithNamespaces() throws Exception { } // *** app-tag - RMApp app2 = rm.submitApp(1*GB); + RMApp app2 = MockRMAppSubmitter.submitWithMemory(1 * GB, rm); // Allocate AM container on nm1 doNodeHeartbeat(nm2); RMAppAttempt app2attempt1 = app2.getCurrentAppAttempt(); @@ -821,7 +896,7 @@ public void testInterAppConstraintsWithNamespaces() throws Exception { } // *** not-self - RMApp app3 = rm.submitApp(1*GB); + RMApp app3 = MockRMAppSubmitter.submitWithMemory(1 * GB, rm); // Allocate AM container on nm1 doNodeHeartbeat(nm3); RMAppAttempt app3attempt1 = app3.getCurrentAppAttempt(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestSchedulingRequestContainerAllocationAsync.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestSchedulingRequestContainerAllocationAsync.java index 9ce03c993b4..e17d768f5fc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestSchedulingRequestContainerAllocationAsync.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestSchedulingRequestContainerAllocationAsync.java @@ -28,6 +28,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; @@ -111,7 +113,15 @@ public RMNodeLabelsManager createNodeLabelManager() { } // app1 -> c - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "c"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nms[0]); // app1 asks for 1000 anti-affinity containers for the same app. It should diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestWorkPreservingRMRestartForNodeLabel.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestWorkPreservingRMRestartForNodeLabel.java index 36ee68e80cb..62668a76334 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestWorkPreservingRMRestartForNodeLabel.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestWorkPreservingRMRestartForNodeLabel.java @@ -33,6 +33,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.TestRMRestart; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; @@ -154,7 +156,15 @@ public RMNodeLabelsManager createNodeLabelManager() { // launch an app to queue a1 (label = x), and check all container will // be allocated in h1 - RMApp app1 = rm1.submitApp(200, "app", "user", null, "a1"); + MockRMAppSubmissionData data2 = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data2); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // request a container. @@ -170,7 +180,15 @@ public RMNodeLabelsManager createNodeLabelManager() { // launch an app to queue b1 (label = y), and check all container will // be allocated in h2 - RMApp app2 = rm1.submitApp(200, "app", "user", null, "b1"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data1); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm2); // request a container. @@ -185,7 +203,15 @@ public RMNodeLabelsManager createNodeLabelManager() { // launch an app to queue c1 (label = ""), and check all container will // be allocated in h3 - RMApp app3 = rm1.submitApp(200, "app", "user", null, "c1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c1") + .withUnmanaged(false) + .build(); + RMApp app3 = MockRMAppSubmitter.submit(rm1, data); MockAM am3 = MockRM.launchAndRegisterAM(app3, rm1, nm3); // request a container. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestPlacementProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestPlacementProcessor.java index 76df0571dd3..bcfad97df4b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestPlacementProcessor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestPlacementProcessor.java @@ -38,6 +38,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; @@ -127,7 +129,15 @@ public void testAntiAffinityPlacement() throws Exception { nm3.registerNode(); nm4.registerNode(); - RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); // Containers with allocationTag 'foo' are restricted to 1 per NODE MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm2, Collections.singletonMap(Collections.singleton("foo"), @@ -176,7 +186,15 @@ public void testMutualAntiAffinityPlacement() throws Exception { nm4.registerNode(); nm5.registerNode(); - RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); // Containers with allocationTag 'foo' are restricted to 1 per NODE Map, PlacementConstraint> pcMap = new HashMap<>(); pcMap.put(Collections.singleton("foo"), @@ -227,7 +245,15 @@ public void testCardinalityPlacement() throws Exception { nm3.registerNode(); nm4.registerNode(); - RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); // Containers with allocationTag 'foo' should not exceed 4 per NODE MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm2, Collections.singletonMap(Collections.singleton("foo"), @@ -280,7 +306,15 @@ public void testAffinityPlacement() throws Exception { nm3.registerNode(); nm4.registerNode(); - RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); // Containers with allocationTag 'foo' should be placed where // containers with allocationTag 'bar' are already running MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm2, @@ -328,7 +362,15 @@ public void testComplexPlacement() throws Exception { nm3.registerNode(); nm4.registerNode(); - RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); Map, PlacementConstraint> constraintMap = new HashMap<>(); // Containers with allocationTag 'bar' should not exceed 1 per NODE constraintMap.put(Collections.singleton("bar"), @@ -399,7 +441,15 @@ public void testSchedulerRejection() throws Exception { nm3.registerNode(); nm4.registerNode(); - RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "a"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); // Containers with allocationTag 'foo' are restricted to 1 per NODE MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm2, Collections.singletonMap( @@ -470,7 +520,15 @@ public void testNodeCapacityRejection() throws Exception { nm3.registerNode(); nm4.registerNode(); - RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); // Containers with allocationTag 'foo' are restricted to 1 per NODE MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm2, Collections.singletonMap( @@ -556,7 +614,15 @@ public void testRePlacementAfterSchedulerRejection() throws Exception { nm4.registerNode(); // Do not register nm5 yet.. - RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); // Containers with allocationTag 'foo' are restricted to 1 per NODE MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm2, Collections.singletonMap( @@ -628,7 +694,15 @@ public void testPlacementRejection() throws Exception { nm3.registerNode(); nm4.registerNode(); - RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); // Containers with allocationTag 'foo' are restricted to 1 per NODE MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm2, Collections.singletonMap( @@ -703,7 +777,15 @@ public void testAndOrPlacement() throws Exception { nm3.registerNode(); nm4.registerNode(); - RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); // Register app1 with following constraints // 1) foo anti-affinity with foo on node diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestApplicationMasterServiceWithFS.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestApplicationMasterServiceWithFS.java index d7ec1621910..9bc6bb4c329 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestApplicationMasterServiceWithFS.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestApplicationMasterServiceWithFS.java @@ -33,6 +33,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; @@ -97,7 +99,10 @@ public void testQueueLevelContainerAllocationFail() throws Exception { MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * GB); // Submit an application - RMApp app1 = rm.submitApp(2 * GB, "queueA"); + MockRMAppSubmissionData data = MockRMAppSubmissionData.Builder.createWithDefaults(2 * GB, rm) + .withQueue("queueA") + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); // kick the scheduling nm1.nodeHeartbeat(true); @@ -139,7 +144,10 @@ private void testFairSchedulerContainerAllocationSuccess(String queueName) MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * GB); // Submit an application - RMApp app1 = rm.submitApp(2 * GB, queueName); + MockRMAppSubmissionData data = MockRMAppSubmissionData.Builder.createWithDefaults(2 * GB, rm) + .withQueue(queueName) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); // kick the scheduling nm1.nodeHeartbeat(true); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index d9f29f1be11..8a509eac1a9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -55,6 +55,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockNodes; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.NodeManager; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; @@ -5217,7 +5219,15 @@ public void testRefreshQueuesWhenRMHA() throws Exception { rm2.getAdminService().transitionToActive(requestInfo); // 4. submit a app to the new added queue "test_queue" - RMApp app = rm2.submitApp(200, "test_app", "user", null, "test_queue"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm2) + .withName("test_app") + .withUser("user") + .withAcls(null) + .withQueue("test_queue") + .withUnmanaged(false) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm2, data); RMAppAttempt attempt0 = app.getCurrentAppAttempt(); nm.nodeHeartbeat(true); MockAM am0 = rm2.sendAMLaunched(attempt0.getAppAttemptId()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java index dc94e5e6d74..ece8c68598f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java @@ -69,6 +69,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockNodes; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.NodeManager; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; @@ -661,7 +662,7 @@ public void testAllocateContainerOnNodeWithoutOffSwitchSpecified() rm.start(); MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * GB); - RMApp app1 = rm.submitApp(2048); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(2048, rm); // kick the scheduling, 2 GB given to AM1, remaining 4GB on nm1 nm1.nodeHeartbeat(true); RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); @@ -692,7 +693,7 @@ public void testFifoScheduling() throws Exception { MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * GB); MockNM nm2 = rm.registerNode("127.0.0.2:5678", 4 * GB); - RMApp app1 = rm.submitApp(2048); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(2048, rm); // kick the scheduling, 2 GB given to AM1, remaining 4GB on nm1 nm1.nodeHeartbeat(true); RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); @@ -702,7 +703,7 @@ public void testFifoScheduling() throws Exception { rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemorySize()); - RMApp app2 = rm.submitApp(2048); + RMApp app2 = MockRMAppSubmitter.submitWithMemory(2048, rm); // kick the scheduling, 2GB given to AM, remaining 2 GB on nm2 nm2.nodeHeartbeat(true); RMAppAttempt attempt2 = app2.getCurrentAppAttempt(); @@ -812,7 +813,7 @@ private void testMinimumAllocation(YarnConfiguration conf, int testAlloc) MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * GB); // Submit an application - RMApp app1 = rm.submitApp(testAlloc); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(testAlloc, rm); // kick the scheduling nm1.nodeHeartbeat(true); @@ -1107,7 +1108,7 @@ public void testResourceOverCommit() throws Exception { MockNM nm1 = rm.registerNode("127.0.0.1:1234", 4 * GB); - RMApp app1 = rm.submitApp(2048); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(2048, rm); // kick the scheduling, 2 GB given to AM1, remaining 2GB on nm1 nm1.nodeHeartbeat(true); RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFairOrderingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFairOrderingPolicy.java index e023e011e5b..b7e0a76e758 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFairOrderingPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFairOrderingPolicy.java @@ -23,6 +23,8 @@ import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; @@ -178,10 +180,42 @@ public void testSizeBasedWeightNotAffectAppActivation() throws Exception { rm.registerNode("h1:1234", 10 * GB); // Submit 4 apps - rm.submitApp(1 * GB, "app", "user", null, "default"); - rm.submitApp(1 * GB, "app", "user", null, "default"); - rm.submitApp(1 * GB, "app", "user", null, "default"); - rm.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data3 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + MockRMAppSubmitter.submit(rm, data3); + MockRMAppSubmissionData data2 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + MockRMAppSubmitter.submit(rm, data2); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + MockRMAppSubmitter.submit(rm, data1); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + MockRMAppSubmitter.submit(rm, data); Assert.assertEquals(1, lq.getNumActiveApplications()); Assert.assertEquals(3, lq.getNumPendingApplications()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java index 53221d88d6e..6a44422d3fa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java @@ -56,6 +56,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.RMSecretManagerService; import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MockRMWithAMS; @@ -129,7 +130,7 @@ public void testTokenExpiry() throws Exception { try { MockNM nm1 = rm.registerNode("localhost:1234", 5120); - RMApp app = rm.submitApp(1024); + RMApp app = MockRMAppSubmitter.submitWithMemory(1024, rm); nm1.nodeHeartbeat(true); @@ -250,7 +251,7 @@ public void testMasterKeyRollOver() throws Exception { try { MockNM nm1 = rm.registerNode("localhost:1234", 5120); - RMApp app = rm.submitApp(1024); + RMApp app = MockRMAppSubmitter.submitWithMemory(1024, rm); nm1.nodeHeartbeat(true); @@ -380,7 +381,7 @@ protected AMRMTokenSecretManager createAMRMTokenSecretManager( }; rm.start(); MockNM nm = rm.registerNode("127.0.0.1:1234", 8000); - RMApp app = rm.submitApp(200); + RMApp app = MockRMAppSubmitter.submitWithMemory(200, rm); MockAM am = MockRM.launchAndRegisterAM(app, rm, nm); AMRMTokenSecretManager spySecretMgr = spySecretMgrRef.get(); // Do allocate. Should not update AMRMToken diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java index 7c77de4d886..1df62966598 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java @@ -58,6 +58,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.MockRMWithCustomAMLauncher; import org.apache.hadoop.yarn.server.resourcemanager.ParameterizedSchedulerTestBase; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; @@ -216,7 +217,7 @@ protected void doSecureLogin() throws IOException { rm.start(); // Submit an app - RMApp app = rm.submitApp(1024); + RMApp app = MockRMAppSubmitter.submitWithMemory(1024, rm); // Set up a node. MockNM nm1 = rm.registerNode("localhost:1234", 3072); @@ -446,7 +447,7 @@ protected void doSecureLogin() throws IOException { rm.start(); // Submit an app - RMApp app = rm.submitApp(1024); + RMApp app = MockRMAppSubmitter.submitWithMemory(1024, rm); // Set up a node. MockNM nm1 = rm.registerNode("localhost:1234", 3072); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java index ee104b41336..fe476aa8b0f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java @@ -88,6 +88,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.TestRMRestart.TestSecurityMockRM; import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore; @@ -356,6 +358,34 @@ static MyToken createTokens(Text renewer) token1.setService(new Text("localhost:0")); return token1; } + + public RMApp submitApp(MockRM rm, Credentials cred, ByteBuffer tokensConf) + throws Exception { + int maxAttempts = rm.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS); + MockRMAppSubmissionData data = MockRMAppSubmissionData.Builder.create() + .withResource(Resource.newInstance(200, 1)) + .withName("app1") + .withUser("user") + .withAcls(null) + .withUnmanaged(false) + .withQueue(null) + .withMaxAppAttempts(maxAttempts) + .withCredentials(cred) + .withAppType(null) + .withWaitForAccepted(true) + .withKeepContainers(false) + .withIsAppIdProvided(false) + .withApplicationId(null) + .withAttemptFailuresValidityInterval(0) + .withLogAggregationContext(null) + .withCancelTokensWhenComplete(true) + .withPriority(Priority.newInstance(0)) + .withAmLabel(null) + .withApplicationTimeouts(null) + .withTokensConf(tokensConf) + .build(); + return MockRMAppSubmitter.submit(rm, data); + } /** @@ -948,9 +978,16 @@ protected DelegationTokenRenewer createDelegationTokenRenewer() { credentials.addToken(userText1, token1); RMApp app = - rm.submitApp(200, "name", "user", - new HashMap(), false, "default", 1, - credentials); + MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm) + .withName("name") + .withUser("user") + .withAcls(new HashMap()) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(1) + .withCredentials(credentials) + .build()); // wait for the initial expiring hdfs token to be removed from allTokens GenericTestUtils.waitFor(new Supplier() { @@ -1029,9 +1066,16 @@ public void testRMRestartWithExpiredToken() throws Exception { MockRM rm1 = new TestSecurityMockRM(yarnConf); MemoryRMStateStore memStore = (MemoryRMStateStore) rm1.getRMStateStore(); rm1.start(); - RMApp app = rm1.submitApp(200, "name", "user", - new HashMap(), false, "default", 1, - credentials); + RMApp app = MockRMAppSubmitter.submit(rm1, + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("name") + .withUser("user") + .withAcls(new HashMap()) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(1) + .withCredentials(credentials) + .build()); // create token2 Text userText2 = new Text("user1"); @@ -1127,7 +1171,7 @@ protected DelegationTokenRenewer createDelegationTokenRenewer() { rm.start(); // submit an app without delegationToken - RMApp app = rm.submitApp(200); + RMApp app = MockRMAppSubmitter.submitWithMemory(200, rm); // wait for the new retrieved hdfs token. GenericTestUtils.waitFor(new Supplier() { @@ -1184,8 +1228,14 @@ public void testAppSubmissionWithPreviousToken() throws Exception{ // submit app1 with a token, set cancelTokenWhenComplete to false; Resource resource = Records.newRecord(Resource.class); resource.setMemorySize(200); - RMApp app1 = rm.submitApp(resource, "name", "user", null, false, null, 2, - credentials, null, true, false, false, null, 0, null, false, null); + MockRMAppSubmissionData data1 = MockRMAppSubmissionData.Builder.createWithDefaults(resource, rm) + .withName("name") + .withUser("user") + .withMaxAppAttempts(2) + .withCredentials(credentials) + .withCancelTokensWhenComplete(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); rm.waitForState(app1.getApplicationId(), RMAppState.RUNNING); DelegationTokenRenewer renewer = @@ -1194,8 +1244,14 @@ public void testAppSubmissionWithPreviousToken() throws Exception{ Assert.assertNotNull(dttr); // submit app2 with the same token, set cancelTokenWhenComplete to true; - RMApp app2 = rm.submitApp(resource, "name", "user", null, false, null, 2, - credentials, null, true, false, false, null, 0, null, true, null); + MockRMAppSubmissionData data = MockRMAppSubmissionData.Builder.createWithDefaults(resource, rm) + .withName("name") + .withUser("user") + .withMaxAppAttempts(2) + .withCredentials(credentials) + .withCancelTokensWhenComplete(true) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm1); rm.waitForState(app2.getApplicationId(), RMAppState.RUNNING); finishAMAndWaitForComplete(app2, rm, nm1, am2, dttr); @@ -1250,17 +1306,28 @@ public void testCancelWithMultipleAppSubmissions() throws Exception{ Resource resource = Records.newRecord(Resource.class); resource.setMemorySize(200); + MockRMAppSubmissionData data2 = MockRMAppSubmissionData.Builder.createWithDefaults(resource, rm) + .withName("name") + .withUser("user") + .withMaxAppAttempts(2) + .withCredentials(credentials) + .build(); RMApp app1 = - rm.submitApp(resource, "name", "user", null, false, null, 2, credentials, - null, true, false, false, null, 0, null, true, null); + MockRMAppSubmitter.submit(rm, data2); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); rm.waitForState(app1.getApplicationId(), RMAppState.RUNNING); DelegationTokenToRenew dttr = renewer.getAllTokens().get(token1); Assert.assertNotNull(dttr); Assert.assertTrue(dttr.referringAppIds.contains(app1.getApplicationId())); - RMApp app2 = rm.submitApp(resource, "name", "user", null, false, null, 2, - credentials, null, true, false, false, null, 0, null, true, null); + MockRMAppSubmissionData data1 = MockRMAppSubmissionData.Builder.createWithDefaults(resource, rm) + .withResource(resource) + .withName("name") + .withUser("user") + .withMaxAppAttempts(2) + .withCredentials(credentials) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm, data1); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm1); rm.waitForState(app2.getApplicationId(), RMAppState.RUNNING); Assert.assertTrue(renewer.getAllTokens().containsKey(token1)); @@ -1276,8 +1343,14 @@ public void testCancelWithMultipleAppSubmissions() throws Exception{ Assert.assertFalse(dttr.isTimerCancelled()); Assert.assertFalse(Renewer.cancelled); - RMApp app3 = rm.submitApp(resource, "name", "user", null, false, null, 2, - credentials, null, true, false, false, null, 0, null, true, null); + MockRMAppSubmissionData data = MockRMAppSubmissionData.Builder.createWithDefaults(resource, rm) + .withResource(resource) + .withName("name") + .withUser("user") + .withMaxAppAttempts(2) + .withCredentials(credentials) + .build(); + RMApp app3 = MockRMAppSubmitter.submit(rm, data); MockAM am3 = MockRM.launchAndRegisterAM(app3, rm, nm1); rm.waitForState(app3.getApplicationId(), RMAppState.RUNNING); Assert.assertTrue(renewer.getAllTokens().containsKey(token1)); @@ -1370,7 +1443,7 @@ public void testRenewTokenUsingTokenConfProvidedByApp() throws Exception{ final int confSize = appConf.size(); // submit app - RMApp app = rm.submitApp(credentials, tokenConf); + RMApp app = submitApp(rm, credentials, tokenConf); GenericTestUtils.waitFor(new Supplier() { public Boolean get() { @@ -1425,7 +1498,7 @@ public void testTokensConfExceedLimit() throws Exception { ByteBuffer tokenConf = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); try { - rm.submitApp(credentials, tokenConf); + submitApp(rm, credentials, tokenConf); Assert.fail(); } catch (Exception e) { e.printStackTrace(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/volume/csi/TestVolumeProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/volume/csi/TestVolumeProcessor.java index 370c527cecb..6f819b65002 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/volume/csi/TestVolumeProcessor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/volume/csi/TestVolumeProcessor.java @@ -36,6 +36,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; @@ -146,7 +148,15 @@ private void writeTmpResourceTypesFile(File tmpFile) throws IOException { @Test (timeout = 10000L) public void testVolumeProvisioning() throws Exception { - RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, mockNMS[0]); Resource resource = Resource.newInstance(1024, 1); ResourceInformation volumeResource = ResourceInformation @@ -199,7 +209,15 @@ public void testVolumeProvisioning() throws Exception { @Test (timeout = 30000L) public void testInvalidRequest() throws Exception { - RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, mockNMS[0]); Resource resource = Resource.newInstance(1024, 1); ResourceInformation volumeResource = ResourceInformation @@ -233,7 +251,15 @@ public void testInvalidRequest() throws Exception { @Test (timeout = 30000L) public void testProvisioningFailures() throws Exception { - RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, mockNMS[0]); CsiAdaptorProtocol mockedClient = Mockito @@ -276,7 +302,13 @@ public void testProvisioningFailures() throws Exception { @Test (timeout = 10000L) public void testVolumeResourceAllocate() throws Exception { - RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default"); + RMApp app1 = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .build()); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, mockNMS[0]); Resource resource = Resource.newInstance(1024, 1); ResourceInformation volumeResource = ResourceInformation diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServiceAppsNodelabel.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServiceAppsNodelabel.java index 3815fb9d06c..54566097040 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServiceAppsNodelabel.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServiceAppsNodelabel.java @@ -35,6 +35,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; @@ -145,7 +147,7 @@ public void testAppsFinished() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); amNodeManager.nodeHeartbeat(true); - RMApp killedApp = rm.submitApp(AM_CONTAINER_MB); + RMApp killedApp = MockRMAppSubmitter.submitWithMemory(AM_CONTAINER_MB, rm); rm.killApp(killedApp.getApplicationId()); WebResource r = resource(); ClientResponse response = @@ -173,7 +175,15 @@ public void testAppsRunning() throws JSONException, Exception { nodeLabelManager.addLabelsToNode( ImmutableMap.of(NodeId.newInstance("h2", 1235), toSet("X"))); - RMApp app1 = rm.submitApp(AM_CONTAINER_MB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(AM_CONTAINER_MB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); nm1.nodeHeartbeat(true); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppAttempts.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppAttempts.java index 07936998ab1..8b7f0d94e08 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppAttempts.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppAttempts.java @@ -30,6 +30,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; @@ -110,7 +112,12 @@ public TestRMWebServicesAppAttempts() { public void testAppAttempts() throws Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(CONTAINER_MB, rm) + .withName("testwordcount") + .withUser("user1") + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); amNodeManager.nodeHeartbeat(true); testAppAttemptsHelper(app1.getApplicationId().toString(), app1, MediaType.APPLICATION_JSON); @@ -121,7 +128,12 @@ public void testAppAttempts() throws Exception { public void testMultipleAppAttempts() throws Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 8192); - RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(CONTAINER_MB, rm) + .withName("testwordcount") + .withUser("user1") + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); MockAM am = MockRM.launchAndRegisterAM(app1, rm, amNodeManager); int maxAppAttempts = rm.getConfig().getInt( YarnConfiguration.RM_AM_MAX_ATTEMPTS, @@ -153,7 +165,7 @@ public void testMultipleAppAttempts() throws Exception { public void testAppAttemptsSlash() throws Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(CONTAINER_MB); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); amNodeManager.nodeHeartbeat(true); testAppAttemptsHelper(app1.getApplicationId().toString() + "/", app1, MediaType.APPLICATION_JSON); @@ -164,7 +176,7 @@ public void testAppAttemptsSlash() throws Exception { public void testAppAttemptsDefault() throws Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(CONTAINER_MB); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); amNodeManager.nodeHeartbeat(true); testAppAttemptsHelper(app1.getApplicationId().toString() + "/", app1, ""); rm.stop(); @@ -174,7 +186,7 @@ public void testAppAttemptsDefault() throws Exception { public void testInvalidAppIdGetAttempts() throws Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app = rm.submitApp(CONTAINER_MB); + RMApp app = MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); @@ -215,7 +227,7 @@ public void testInvalidAppIdGetAttempts() throws Exception { public void testInvalidAppAttemptId() throws Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app = rm.submitApp(CONTAINER_MB); + RMApp app = MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); @@ -256,7 +268,12 @@ public void testInvalidAppAttemptId() throws Exception { public void testNonexistAppAttempts() throws Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - rm.submitApp(CONTAINER_MB, "testwordcount", "user1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(CONTAINER_MB, rm) + .withName("testwordcount") + .withUser("user1") + .build(); + MockRMAppSubmitter.submit(rm, data); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); @@ -322,7 +339,12 @@ public void testAppAttemptsXML() throws Exception { rm.start(); String user = "user1"; MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", user); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(CONTAINER_MB, rm) + .withName("testwordcount") + .withUser(user) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("cluster") diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppCustomResourceTypes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppCustomResourceTypes.java index bc3ab208c2a..7d1c18d7edf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppCustomResourceTypes.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppCustomResourceTypes.java @@ -27,6 +27,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; @@ -114,7 +116,12 @@ private WebResource getWebResourcePathForApp(RMApp app1, WebResource r) { public void testRunningAppXml() throws Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(CONTAINER_MB, rm) + .withName("testwordcount") + .withUser("user1") + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, amNodeManager); am1.allocate("*", 2048, 1, new ArrayList<>()); amNodeManager.nodeHeartbeat(true); @@ -142,7 +149,12 @@ public void testRunningAppXml() throws Exception { public void testRunningAppJson() throws Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(CONTAINER_MB, rm) + .withName("testwordcount") + .withUser("user1") + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, amNodeManager); am1.allocate("*", 2048, 1, new ArrayList<>()); amNodeManager.nodeHeartbeat(true); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index 6541eecf364..ad4e58db87c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -39,6 +39,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; @@ -131,7 +133,7 @@ public TestRMWebServicesApps() { public void testApps() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(CONTAINER_MB); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); amNodeManager.nodeHeartbeat(true); testAppsHelper("apps", app1, MediaType.APPLICATION_JSON); rm.stop(); @@ -141,7 +143,7 @@ public void testApps() throws JSONException, Exception { public void testAppsSlash() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(CONTAINER_MB); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); amNodeManager.nodeHeartbeat(true); testAppsHelper("apps/", app1, MediaType.APPLICATION_JSON); rm.stop(); @@ -151,7 +153,7 @@ public void testAppsSlash() throws JSONException, Exception { public void testAppsDefault() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(CONTAINER_MB); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); amNodeManager.nodeHeartbeat(true); testAppsHelper("apps/", app1, ""); rm.stop(); @@ -161,7 +163,12 @@ public void testAppsDefault() throws JSONException, Exception { public void testAppsXML() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(CONTAINER_MB, rm) + .withName("testwordcount") + .withUser("user1") + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("cluster") @@ -187,7 +194,12 @@ public void testAppsXML() throws JSONException, Exception { public void testRunningApp() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(CONTAINER_MB, rm) + .withName("testwordcount") + .withUser("user1") + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, amNodeManager); am1.allocate("*", 4096, 1, new ArrayList<>()); amNodeManager.nodeHeartbeat(true); @@ -218,8 +230,18 @@ public void testRunningApp() throws JSONException, Exception { public void testAppsXMLMulti() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - rm.submitApp(CONTAINER_MB, "testwordcount", "user1"); - rm.submitApp(2048, "testwordcount2", "user1"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(CONTAINER_MB, rm) + .withName("testwordcount") + .withUser("user1") + .build(); + MockRMAppSubmitter.submit(rm, data1); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(2048, rm) + .withName("testwordcount2") + .withUser("user1") + .build(); + MockRMAppSubmitter.submit(rm, data); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); @@ -269,7 +291,7 @@ public void testAppsHelper(String path, RMApp app, String media, public void testAppsQueryState() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(CONTAINER_MB); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); @@ -293,8 +315,8 @@ public void testAppsQueryState() throws JSONException, Exception { public void testAppsQueryStates() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - rm.submitApp(CONTAINER_MB); - RMApp killedApp = rm.submitApp(CONTAINER_MB); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); + RMApp killedApp = MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); rm.killApp(killedApp.getApplicationId()); amNodeManager.nodeHeartbeat(true); @@ -344,8 +366,8 @@ public void testAppsQueryStates() throws JSONException, Exception { public void testAppsQueryStatesComma() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - rm.submitApp(CONTAINER_MB); - RMApp killedApp = rm.submitApp(CONTAINER_MB); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); + RMApp killedApp = MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); rm.killApp(killedApp.getApplicationId()); amNodeManager.nodeHeartbeat(true); @@ -395,7 +417,7 @@ public void testAppsQueryStatesComma() throws JSONException, Exception { public void testAppsQueryStatesNone() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - rm.submitApp(CONTAINER_MB); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); @@ -416,7 +438,7 @@ public void testAppsQueryStatesNone() throws JSONException, Exception { public void testAppsQueryStateNone() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - rm.submitApp(CONTAINER_MB); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); @@ -437,7 +459,7 @@ public void testAppsQueryStateNone() throws JSONException, Exception { public void testAppsQueryStatesInvalid() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - rm.submitApp(CONTAINER_MB); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); @@ -475,7 +497,7 @@ public void testAppsQueryStatesInvalid() throws JSONException, Exception { public void testAppsQueryStateInvalid() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - rm.submitApp(CONTAINER_MB); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); @@ -513,7 +535,7 @@ public void testAppsQueryStateInvalid() throws JSONException, Exception { public void testAppsQueryFinalStatus() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(CONTAINER_MB); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); @@ -538,7 +560,7 @@ public void testAppsQueryFinalStatus() throws JSONException, Exception { public void testAppsQueryFinalStatusNone() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - rm.submitApp(CONTAINER_MB); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); @@ -558,7 +580,7 @@ public void testAppsQueryFinalStatusNone() throws JSONException, Exception { public void testAppsQueryFinalStatusInvalid() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - rm.submitApp(CONTAINER_MB); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); @@ -597,8 +619,8 @@ public void testAppsQueryFinalStatusInvalid() throws JSONException, Exception { public void testAppsQueryUser() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - rm.submitApp(CONTAINER_MB); - rm.submitApp(CONTAINER_MB); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); @@ -626,8 +648,8 @@ public void testAppsQueryUser() throws JSONException, Exception { public void testAppsQueryQueue() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - rm.submitApp(CONTAINER_MB); - rm.submitApp(CONTAINER_MB); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); @@ -650,8 +672,8 @@ public void testAppsQueryQueue() throws JSONException, Exception { public void testAppsQueryQueueAndStateTwoFinishedApps() throws Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(CONTAINER_MB); - RMApp app2 = rm.submitApp(CONTAINER_MB); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); + RMApp app2 = MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); amNodeManager.nodeHeartbeat(true); finishApp(amNodeManager, app1); @@ -686,8 +708,8 @@ public void testAppsQueryQueueAndStateTwoFinishedApps() throws Exception { public void testAppsQueryQueueAndStateOneFinishedApp() throws Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp finishedApp = rm.submitApp(CONTAINER_MB); - RMApp runningApp = rm.submitApp(CONTAINER_MB); + RMApp finishedApp = MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); + RMApp runningApp = MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); amNodeManager.nodeHeartbeat(true); finishApp(amNodeManager, finishedApp); @@ -722,8 +744,8 @@ public void testAppsQueryQueueAndStateOneFinishedApp() throws Exception { public void testAppsQueryQueueOneFinishedApp() throws Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp finishedApp = rm.submitApp(CONTAINER_MB); - RMApp runningApp = rm.submitApp(CONTAINER_MB); + RMApp finishedApp = MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); + RMApp runningApp = MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); amNodeManager.nodeHeartbeat(true); finishApp(amNodeManager, finishedApp); @@ -757,9 +779,9 @@ public void testAppsQueryQueueOneFinishedApp() throws Exception { public void testAppsQueryLimit() throws JSONException, Exception { rm.start(); rm.registerNode("127.0.0.1:1234", 2048); - rm.submitApp(CONTAINER_MB); - rm.submitApp(CONTAINER_MB); - rm.submitApp(CONTAINER_MB); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("cluster") .path("apps").queryParam("limit", "2") @@ -781,9 +803,9 @@ public void testAppsQueryStartBegin() throws JSONException, Exception { long start = System.currentTimeMillis(); Thread.sleep(1); rm.registerNode("127.0.0.1:1234", 2048); - rm.submitApp(CONTAINER_MB); - rm.submitApp(CONTAINER_MB); - rm.submitApp(CONTAINER_MB); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("cluster") .path("apps").queryParam("startedTimeBegin", String.valueOf(start)) @@ -803,11 +825,11 @@ public void testAppsQueryStartBegin() throws JSONException, Exception { public void testAppsQueryStartBeginSome() throws JSONException, Exception { rm.start(); rm.registerNode("127.0.0.1:1234", 2048); - rm.submitApp(CONTAINER_MB); - rm.submitApp(CONTAINER_MB); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); long start = System.currentTimeMillis(); Thread.sleep(1); - rm.submitApp(CONTAINER_MB); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("cluster") .path("apps").queryParam("startedTimeBegin", String.valueOf(start)) @@ -829,9 +851,9 @@ public void testAppsQueryStartEnd() throws JSONException, Exception { rm.registerNode("127.0.0.1:1234", 2048); long end = System.currentTimeMillis(); Thread.sleep(1); - rm.submitApp(CONTAINER_MB); - rm.submitApp(CONTAINER_MB); - rm.submitApp(CONTAINER_MB); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("cluster") .path("apps").queryParam("startedTimeEnd", String.valueOf(end)) @@ -851,11 +873,11 @@ public void testAppsQueryStartBeginEnd() throws JSONException, Exception { rm.registerNode("127.0.0.1:1234", 2048); long start = System.currentTimeMillis(); Thread.sleep(1); - rm.submitApp(CONTAINER_MB); - rm.submitApp(CONTAINER_MB); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); long end = System.currentTimeMillis(); Thread.sleep(1); - rm.submitApp(CONTAINER_MB); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("cluster") .path("apps").queryParam("startedTimeBegin", String.valueOf(start)) @@ -878,11 +900,11 @@ public void testAppsQueryFinishBegin() throws JSONException, Exception { MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); long start = System.currentTimeMillis(); Thread.sleep(1); - RMApp app1 = rm.submitApp(CONTAINER_MB); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); amNodeManager.nodeHeartbeat(true); finishApp(amNodeManager, app1); - rm.submitApp(CONTAINER_MB); - rm.submitApp(CONTAINER_MB); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("cluster") @@ -913,13 +935,13 @@ private void finishApp(MockNM amNodeManager, RMApp app) throws Exception { public void testAppsQueryFinishEnd() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(CONTAINER_MB); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); amNodeManager.nodeHeartbeat(true); // finish App finishApp(amNodeManager, app1); - rm.submitApp(CONTAINER_MB); - rm.submitApp(CONTAINER_MB); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); long end = System.currentTimeMillis(); WebResource r = resource(); @@ -943,13 +965,13 @@ public void testAppsQueryFinishBeginEnd() throws JSONException, Exception { MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); long start = System.currentTimeMillis(); Thread.sleep(1); - RMApp app1 = rm.submitApp(CONTAINER_MB); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); amNodeManager.nodeHeartbeat(true); // finish App finishApp(amNodeManager, app1); - rm.submitApp(CONTAINER_MB); - rm.submitApp(CONTAINER_MB); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); long end = System.currentTimeMillis(); WebResource r = resource(); @@ -973,15 +995,35 @@ public void testAppsQueryAppTypes() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); Thread.sleep(1); - RMApp app1 = rm.submitApp(CONTAINER_MB); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); amNodeManager.nodeHeartbeat(true); // finish App finishApp(amNodeManager, app1); - rm.submitApp(CONTAINER_MB, "", UserGroupInformation.getCurrentUser() - .getShortUserName(), null, false, null, 2, null, "MAPREDUCE"); - rm.submitApp(CONTAINER_MB, "", UserGroupInformation.getCurrentUser() - .getShortUserName(), null, false, null, 2, null, "NON-YARN"); + MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(CONTAINER_MB, rm) + .withName("") + .withUser(UserGroupInformation.getCurrentUser() + .getShortUserName()) + .withAcls(null) + .withUnmanaged(false) + .withQueue(null) + .withMaxAppAttempts(2) + .withCredentials(null) + .withAppType("MAPREDUCE") + .build()); + MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(CONTAINER_MB, rm) + .withName("") + .withUser(UserGroupInformation.getCurrentUser() + .getShortUserName()) + .withAcls(null) + .withUnmanaged(false) + .withQueue(null) + .withMaxAppAttempts(2) + .withCredentials(null) + .withAppType("NON-YARN") + .build()); WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("cluster") @@ -1164,7 +1206,7 @@ public void testAppsQueryWithInvaildDeselects() try { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - rm.submitApp(CONTAINER_MB); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("cluster") @@ -1197,7 +1239,7 @@ public void testAppsQueryWithDeselects() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - rm.submitApp(CONTAINER_MB); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); @@ -1303,15 +1345,45 @@ public void testAppStatistics() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 4096); Thread.sleep(1); - RMApp app1 = rm.submitApp(CONTAINER_MB, "", UserGroupInformation.getCurrentUser() - .getShortUserName(), null, false, null, 2, null, "MAPREDUCE"); + RMApp app1 = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(CONTAINER_MB, rm) + .withName("") + .withUser(UserGroupInformation.getCurrentUser() + .getShortUserName()) + .withAcls(null) + .withUnmanaged(false) + .withQueue(null) + .withMaxAppAttempts(2) + .withCredentials(null) + .withAppType("MAPREDUCE") + .build()); amNodeManager.nodeHeartbeat(true); finishApp(amNodeManager, app1); - rm.submitApp(CONTAINER_MB, "", UserGroupInformation.getCurrentUser() - .getShortUserName(), null, false, null, 2, null, "MAPREDUCE"); - rm.submitApp(CONTAINER_MB, "", UserGroupInformation.getCurrentUser() - .getShortUserName(), null, false, null, 2, null, "OTHER"); + MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(CONTAINER_MB, rm) + .withName("") + .withUser(UserGroupInformation.getCurrentUser() + .getShortUserName()) + .withAcls(null) + .withUnmanaged(false) + .withQueue(null) + .withMaxAppAttempts(2) + .withCredentials(null) + .withAppType("MAPREDUCE") + .build()); + MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(CONTAINER_MB, rm) + .withName("") + .withUser(UserGroupInformation.getCurrentUser() + .getShortUserName()) + .withAcls(null) + .withUnmanaged(false) + .withQueue(null) + .withMaxAppAttempts(2) + .withCredentials(null) + .withAppType("OTHER") + .build()); // zero type, zero state WebResource r = resource(); @@ -1464,7 +1536,12 @@ public void testAppStatistics() throws JSONException, Exception { public void testSingleApp() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(CONTAINER_MB, rm) + .withName("testwordcount") + .withUser("user1") + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); amNodeManager.nodeHeartbeat(true); testSingleAppsHelper(app1.getApplicationId().toString(), app1, MediaType.APPLICATION_JSON); @@ -1475,7 +1552,12 @@ public void testSingleApp() throws JSONException, Exception { public void testUnmarshalAppInfo() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(CONTAINER_MB, rm) + .withName("testwordcount") + .withUser("user1") + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); @@ -1499,7 +1581,7 @@ public void testUnmarshalAppInfo() throws JSONException, Exception { public void testSingleAppsSlash() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(CONTAINER_MB); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); amNodeManager.nodeHeartbeat(true); testSingleAppsHelper(app1.getApplicationId().toString() + "/", app1, MediaType.APPLICATION_JSON); @@ -1510,7 +1592,7 @@ public void testSingleAppsSlash() throws JSONException, Exception { public void testSingleAppsDefault() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(CONTAINER_MB); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); amNodeManager.nodeHeartbeat(true); testSingleAppsHelper(app1.getApplicationId().toString() + "/", app1, ""); rm.stop(); @@ -1520,7 +1602,7 @@ public void testSingleAppsDefault() throws JSONException, Exception { public void testInvalidApp() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - rm.submitApp(CONTAINER_MB); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); @@ -1559,7 +1641,12 @@ public void testInvalidApp() throws JSONException, Exception { public void testNonexistApp() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - rm.submitApp(CONTAINER_MB, "testwordcount", "user1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(CONTAINER_MB, rm) + .withName("testwordcount") + .withUser("user1") + .build(); + MockRMAppSubmitter.submit(rm, data); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); @@ -1610,7 +1697,12 @@ public void testSingleAppsHelper(String path, RMApp app, String media) public void testSingleAppsXML() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(CONTAINER_MB, rm) + .withName("testwordcount") + .withUser("user1") + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("cluster") diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsCustomResourceTypes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsCustomResourceTypes.java index d8f461ebda9..62043e83ab9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsCustomResourceTypes.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsCustomResourceTypes.java @@ -29,6 +29,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; @@ -116,7 +118,12 @@ public TestRMWebServicesAppsCustomResourceTypes() { public void testRunningAppsXml() throws Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(CONTAINER_MB, rm) + .withName("testwordcount") + .withUser("user1") + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, amNodeManager); am1.allocate("*", 2048, 1, new ArrayList<>()); amNodeManager.nodeHeartbeat(true); @@ -147,7 +154,12 @@ public void testRunningAppsXml() throws Exception { public void testRunningAppsJson() throws Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(CONTAINER_MB, rm) + .withName("testwordcount") + .withUser("user1") + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, amNodeManager); am1.allocate("*", 2048, 1, new ArrayList<>()); amNodeManager.nodeHeartbeat(true); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java index 5d99e0d169d..5c6f996ae89 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java @@ -74,6 +74,8 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; @@ -339,7 +341,12 @@ public void testSingleAppState() throws Exception { String[] mediaTypes = { MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }; for (String mediaType : mediaTypes) { - RMApp app = rm.submitApp(CONTAINER_MB, "", webserviceUserName); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(CONTAINER_MB, rm) + .withName("") + .withUser(webserviceUserName) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); amNodeManager.nodeHeartbeat(true); ClientResponse response = this @@ -366,7 +373,12 @@ public void testSingleAppKill() throws Exception { String diagnostic = "message1"; for (String mediaType : mediaTypes) { for (MediaType contentType : contentTypes) { - RMApp app = rm.submitApp(CONTAINER_MB, "", webserviceUserName); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(CONTAINER_MB, rm) + .withName("") + .withUser(webserviceUserName) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); amNodeManager.nodeHeartbeat(true); AppState targetState = @@ -457,7 +469,12 @@ public void testSingleAppKillInvalidState() throws Exception { for (String mediaType : mediaTypes) { for (MediaType contentType : contentTypes) { for (String targetStateString : targetStates) { - RMApp app = rm.submitApp(CONTAINER_MB, "", webserviceUserName); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(CONTAINER_MB, rm) + .withName("") + .withUser(webserviceUserName) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); amNodeManager.nodeHeartbeat(true); ClientResponse response; AppState targetState = new AppState(targetStateString); @@ -564,7 +581,12 @@ public void testSingleAppKillUnauthorized() throws Exception { String[] mediaTypes = { MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }; for (String mediaType : mediaTypes) { - RMApp app = rm.submitApp(CONTAINER_MB, "test", "someuser"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(CONTAINER_MB, rm) + .withName("test") + .withUser("someuser") + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); amNodeManager.nodeHeartbeat(true); ClientResponse response = this @@ -1036,7 +1058,12 @@ public void testGetAppQueue() throws Exception { String[] contentTypes = { MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }; for (String contentType : contentTypes) { - RMApp app = rm.submitApp(CONTAINER_MB, "", webserviceUserName); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(CONTAINER_MB, rm) + .withName("") + .withUser(webserviceUserName) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); amNodeManager.nodeHeartbeat(true); ClientResponse response = this @@ -1090,7 +1117,12 @@ public void testUpdateAppPriority() throws Exception { { MediaType.APPLICATION_JSON_TYPE, MediaType.APPLICATION_XML_TYPE }; for (String mediaType : mediaTypes) { for (MediaType contentType : contentTypes) { - RMApp app = rm.submitApp(CONTAINER_MB, "", webserviceUserName); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(CONTAINER_MB, rm) + .withName("") + .withUser(webserviceUserName) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data1); amNodeManager.nodeHeartbeat(true); int modifiedPriority = 8; AppPriority priority = new AppPriority(modifiedPriority); @@ -1130,7 +1162,12 @@ public void testUpdateAppPriority() throws Exception { } // check unauthorized - app = rm.submitApp(CONTAINER_MB, "", "someuser"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(CONTAINER_MB, rm) + .withName("") + .withUser("someuser") + .build(); + app = MockRMAppSubmitter.submit(rm, data); amNodeManager.nodeHeartbeat(true); response = this .constructWebResource("apps", app.getApplicationId().toString(), @@ -1171,7 +1208,12 @@ public void testAppMove() throws Exception { { MediaType.APPLICATION_JSON_TYPE, MediaType.APPLICATION_XML_TYPE }; for (String mediaType : mediaTypes) { for (MediaType contentType : contentTypes) { - RMApp app = rm.submitApp(CONTAINER_MB, "", webserviceUserName); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(CONTAINER_MB, rm) + .withName("") + .withUser(webserviceUserName) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data1); amNodeManager.nodeHeartbeat(true); AppQueue targetQueue = new AppQueue("test"); Object entity; @@ -1204,7 +1246,12 @@ public void testAppMove() throws Exception { Assert.assertEquals(expectedQueue, app.getQueue()); // check unauthorized - app = rm.submitApp(CONTAINER_MB, "", "someuser"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(CONTAINER_MB, rm) + .withName("") + .withUser("someuser") + .build(); + app = MockRMAppSubmitter.submit(rm, data); amNodeManager.nodeHeartbeat(true); response = this @@ -1312,7 +1359,12 @@ public void testUpdateAppTimeout() throws Exception { for (String mediaType : mediaTypes) { for (MediaType contentType : contentTypes) { // application submitted without timeout - RMApp app = rm.submitApp(CONTAINER_MB, "", webserviceUserName); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(CONTAINER_MB, rm) + .withName("") + .withUser(webserviceUserName) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); ClientResponse response = this.constructWebResource("apps", app.getApplicationId().toString(), diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java index 0fe4cb5f3b3..29109a04203 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java @@ -33,6 +33,8 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; @@ -530,8 +532,24 @@ public void testPerUserResourcesXML() throws Exception { //Start RM so that it accepts app submissions rm.start(); try { - rm.submitApp(10, "app1", "user1", null, "b1"); - rm.submitApp(20, "app2", "user2", null, "b1"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(10, rm) + .withName("app1") + .withUser("user1") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + MockRMAppSubmitter.submit(rm, data1); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(20, rm) + .withName("app2") + .withUser("user2") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + MockRMAppSubmitter.submit(rm, data); //Get the XML from ws/v1/cluster/scheduler WebResource r = resource(); @@ -611,8 +629,24 @@ public void testPerUserResourcesJSON() throws Exception { //Start RM so that it accepts app submissions rm.start(); try { - rm.submitApp(10, "app1", "user1", null, "b1"); - rm.submitApp(20, "app2", "user2", null, "b1"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(10, rm) + .withName("app1") + .withUser("user1") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + MockRMAppSubmitter.submit(rm, data1); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(20, rm) + .withName("app2") + .withUser("user2") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + MockRMAppSubmitter.submit(rm, data); //Get JSON WebResource r = resource(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesContainers.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesContainers.java index 76e7f7b9129..0c2e1b41f6f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesContainers.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesContainers.java @@ -31,6 +31,8 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; @@ -107,7 +109,8 @@ public TestRMWebServicesContainers() { public void testSignalContainer() throws Exception { rm.start(); MockNM nm = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app = rm.submitApp(1024); + RMApp app = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm).build()); nm.nodeHeartbeat(true); MockRM .waitForState(app.getCurrentAppAttempt(), RMAppAttemptState.ALLOCATED); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java index 7c3ec6cdb3d..8f1f78d8ae5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java @@ -50,6 +50,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; @@ -277,7 +279,14 @@ public void testPartitionInSchedulerActivities() throws Exception { nm2.registerNode(); try { - RMApp app1 = rm.submitApp(1024, "app1", "user1", null, QUEUE_B, LABEL_LX); + RMApp app1 = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm) + .withName("app1") + .withUser("user1") + .withAcls(null) + .withQueue(QUEUE_B) + .withAmLabel(LABEL_LX) + .build()); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); am1.allocate(Arrays.asList( ResourceRequest.newBuilder().priority(Priority.UNDEFINED) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java index 1942708b88e..51b0ab57664 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java @@ -38,6 +38,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; @@ -104,7 +106,15 @@ public void testAssignMultipleContainersPerNodeHeartbeat() nm.registerNode(); try { - RMApp app1 = rm.submitApp(10, "app1", "user1", null, "b1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(10, rm) + .withName("app1") + .withUser("user1") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm); am1.allocate(Arrays.asList(ResourceRequest .newInstance(Priority.UNDEFINED, "127.0.0.1", @@ -160,7 +170,15 @@ public void testAssignWithoutAvailableResource() throws Exception { nm.registerNode(); try { - RMApp app1 = rm.submitApp(1024, "app1", "user1", null, "b1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm) + .withName("app1") + .withUser("user1") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm); am1.allocate(Arrays.asList(ResourceRequest .newInstance(Priority.UNDEFINED, "127.0.0.1", @@ -248,7 +266,15 @@ public void testWrongNodeId() throws Exception { nm.registerNode(); try { - RMApp app1 = rm.submitApp(1024, "app1", "user1", null, "b1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm) + .withName("app1") + .withUser("user1") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm); am1.allocate(Arrays.asList(ResourceRequest .newInstance(Priority.UNDEFINED, "127.0.0.1", @@ -300,10 +326,26 @@ public void testReserveNewContainer() throws Exception { nm2.registerNode(); try { - RMApp app1 = rm.submitApp(10, "app1", "user1", null, "b1"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(10, rm) + .withName("app1") + .withUser("user1") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); - RMApp app2 = rm.submitApp(10, "app2", "user1", null, "b2"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(10, rm) + .withName("app2") + .withUser("user1") + .withAcls(null) + .withQueue("b2") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm2); am1.allocate(Arrays.asList(ResourceRequest @@ -420,7 +462,15 @@ public void testActivityJSON() throws Exception { nm.registerNode(); try { - RMApp app1 = rm.submitApp(10, "app1", "user1", null, "b1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(10, rm) + .withName("app1") + .withUser("user1") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); //Get JSON WebResource r = resource(); @@ -470,7 +520,15 @@ public void testAppActivityJSON() throws Exception { nm.registerNode(); try { - RMApp app1 = rm.submitApp(10, "app1", "user1", null, "b1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(10, rm) + .withName("app1") + .withUser("user1") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); //Get JSON WebResource r = resource().path(RMWSConsts.RM_WEB_SERVICE_PATH) @@ -524,7 +582,15 @@ public void testAppAssignMultipleContainersPerNodeHeartbeat() nm.registerNode(); try { - RMApp app1 = rm.submitApp(1024, "app1", "user1", null, "b1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm) + .withName("app1") + .withUser("user1") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm); am1.allocate(Arrays.asList(ResourceRequest .newInstance(Priority.UNDEFINED, "127.0.0.1", @@ -570,7 +636,15 @@ public void testAppAssignWithoutAvailableResource() throws Exception { nm.registerNode(); try { - RMApp app1 = rm.submitApp(1024, "app1", "user1", null, "b1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm) + .withName("app1") + .withUser("user1") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm); am1.allocate(Arrays.asList(ResourceRequest .newInstance(Priority.UNDEFINED, "127.0.0.1", @@ -604,7 +678,15 @@ public void testAppNoNM() throws Exception { rm.start(); try { - RMApp app1 = rm.submitApp(1024, "app1", "user1", null, "b1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm) + .withName("app1") + .withUser("user1") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); //Get JSON WebResource r = resource().path(RMWSConsts.RM_WEB_SERVICE_PATH) @@ -635,10 +717,26 @@ public void testAppReserveNewContainer() throws Exception { nm2.registerNode(); try { - RMApp app1 = rm.submitApp(10, "app1", "user1", null, "b1"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(10, rm) + .withName("app1") + .withUser("user1") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); - RMApp app2 = rm.submitApp(10, "app2", "user1", null, "b2"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(10, rm) + .withName("app2") + .withUser("user1") + .withAcls(null) + .withQueue("b2") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm2); am1.allocate(Arrays.asList(ResourceRequest @@ -693,7 +791,13 @@ public void testInsufficientResourceDiagnostic() throws Exception { MockNM nm2 = rm.registerNode("127.0.0.2:1234", 8 * 1024); try { - RMApp app1 = rm.submitApp(512, "app1", "user1", null, "b1"); + RMApp app1 = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(512, rm) + .withName("app1") + .withUser("user1") + .withAcls(null) + .withQueue("b1") + .build()); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); WebResource r = resource(); @@ -750,7 +854,13 @@ public void testPlacementConstraintDiagnostic() throws Exception { MockNM nm1 = rm.registerNode("127.0.0.1:1234", 4 * 1024); try { - RMApp app1 = rm.submitApp(512, "app1", "user1", null, "b1"); + RMApp app1 = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(512, rm) + .withName("app1") + .withUser("user1") + .withAcls(null) + .withQueue("b1") + .build()); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); // init scheduling request @@ -813,7 +923,13 @@ public void testAppInsufficientResourceDiagnostic() throws Exception { MockNM nm2 = rm.registerNode("127.0.0.2:1234", 8 * 1024); try { - RMApp app1 = rm.submitApp(512, "app1", "user1", null, "b1"); + RMApp app1 = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(512, rm) + .withName("app1") + .withUser("user1") + .withAcls(null) + .withQueue("b1") + .build()); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); WebResource r = resource().path(RMWSConsts.RM_WEB_SERVICE_PATH) @@ -859,7 +975,13 @@ public void testAppPlacementConstraintDiagnostic() throws Exception { MockNM nm2 = rm.registerNode("127.0.0.2:1234", 8 * 1024); try { - RMApp app1 = rm.submitApp(512, "app1", "user1", null, "b1"); + RMApp app1 = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(512, rm) + .withName("app1") + .withUser("user1") + .withAcls(null) + .withQueue("b1") + .build()); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); WebResource r = resource().path(RMWSConsts.RM_WEB_SERVICE_PATH) @@ -911,7 +1033,13 @@ public void testAppFilterByRequestPrioritiesAndAllocationRequestIds() MockNM nm1 = rm.registerNode("127.0.0.1:1234", 8 * 1024); try { - RMApp app1 = rm.submitApp(512, "app1", "user1", null, "b1"); + RMApp app1 = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(512, rm) + .withName("app1") + .withUser("user1") + .withAcls(null) + .withQueue("b1") + .build()); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); WebResource r = resource().path(RMWSConsts.RM_WEB_SERVICE_PATH) @@ -1042,7 +1170,13 @@ public void testAppLimit() throws Exception { MockNM nm1 = rm.registerNode("127.0.0.1:1234", 4 * 1024); MockNM nm2 = rm.registerNode("127.0.0.2:1234", 8 * 1024); try { - RMApp app1 = rm.submitApp(512, "app1", "user1", null, "b1"); + RMApp app1 = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(512, rm) + .withName("app1") + .withUser("user1") + .withAcls(null) + .withQueue("b1") + .build()); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); WebResource r = resource().path(RMWSConsts.RM_WEB_SERVICE_PATH) @@ -1110,7 +1244,13 @@ public void testAppActions() throws Exception { CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); MockNM nm1 = rm.registerNode("127.0.0.1:1234", 8 * 1024); try { - RMApp app1 = rm.submitApp(512, "app1", "user1", null, "b1"); + RMApp app1 = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(512, rm) + .withName("app1") + .withUser("user1") + .withAcls(null) + .withQueue("b1") + .build()); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); // am1 asks for 10 * 512MB container am1.allocate("*", 512, 10, new ArrayList<>()); @@ -1212,7 +1352,13 @@ public void testAppSummary() throws Exception { MockNM nm3 = rm.registerNode("127.0.0.3:1234", 4 * 1024); try { - RMApp app1 = rm.submitApp(5120, "app1", "user1", null, "b1"); + RMApp app1 = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(5120, rm) + .withName("app1") + .withUser("user1") + .withAcls(null) + .withQueue("b1") + .build()); WebResource r = resource().path(RMWSConsts.RM_WEB_SERVICE_PATH) .path(ActivitiesTestUtils.format(RMWSConsts.SCHEDULER_APP_ACTIVITIES, @@ -1278,7 +1424,13 @@ public void testNodeSkippedBecauseOfRelaxLocality() throws Exception { nm2.registerNode(); try { - RMApp app1 = rm.submitApp(10, "app1", "user1", null, "b1"); + RMApp app1 = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(10, rm) + .withName("app1") + .withUser("user1") + .withAcls(null) + .withQueue("b1") + .build()); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); am1.allocate(Arrays.asList( @@ -1363,7 +1515,13 @@ public void testQueueSkippedBecauseOfHeadroom() throws Exception { nm2.registerNode(); try { - RMApp app1 = rm.submitApp(10, "app1", "user1", null, "a1a"); + RMApp app1 = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(10, rm) + .withName("app1") + .withUser("user1") + .withAcls(null) + .withQueue("a1a") + .build()); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); am1.allocate(Arrays.asList( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivitiesWithMultiNodesEnabled.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivitiesWithMultiNodesEnabled.java index 85b418ce6d8..3a877262cea 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivitiesWithMultiNodesEnabled.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivitiesWithMultiNodesEnabled.java @@ -31,6 +31,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; @@ -164,7 +166,13 @@ public void testAssignContainer() throws Exception { nm.registerNode(); try { - RMApp app1 = rm.submitApp(1024, "app1", "user1", null, "b"); + RMApp app1 = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm) + .withName("app1") + .withUser("user1") + .withAcls(null) + .withQueue("b") + .build()); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm); am1.allocate(Arrays.asList(ResourceRequest .newInstance(Priority.UNDEFINED, "127.0.0.1", @@ -256,7 +264,13 @@ public void testAppAssignContainer() throws Exception { MockNM nm2 = rm.registerNode("127.0.0.2:1234", 2 * 1024); try { - RMApp app1 = rm.submitApp(1024, "app1", "user1", null, "b"); + RMApp app1 = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm) + .withName("app1") + .withUser("user1") + .withAcls(null) + .withQueue("b1") + .build()); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); am1.allocate(Arrays.asList(ResourceRequest @@ -318,10 +332,22 @@ public void testInsufficientResourceDiagnostic() throws Exception { MockNM nm4 = rm.registerNode("127.0.0.4:1234", 2 * 1024); try { - RMApp app1 = rm.submitApp(3072, "app1", "user1", null, "b"); + RMApp app1 = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(3072, rm) + .withName("app1") + .withUser("user1") + .withAcls(null) + .withQueue("b") + .build()); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); - RMApp app2 = rm.submitApp(1024, "app2", "user1", null, "b"); + RMApp app2 = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm) + .withName("app2") + .withUser("user1") + .withAcls(null) + .withQueue("b") + .build()); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm1); WebResource r = resource(); @@ -392,7 +418,13 @@ public void testAppInsufficientResourceDiagnostic() throws Exception { MockNM nm4 = rm.registerNode("127.0.0.4:1234", 2 * 1024); try { - RMApp app1 = rm.submitApp(3072, "app1", "user1", null, "b"); + RMApp app1 = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(3072, rm) + .withName("app1") + .withUser("user1") + .withAcls(null) + .withQueue("b") + .build()); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); WebResource r = resource().path(RMWSConsts.RM_WEB_SERVICE_PATH) @@ -465,7 +497,13 @@ public void testGroupByDiagnostics() throws Exception { MockNM nm4 = rm.registerNode("127.0.0.4:1234", 2 * 1024); try { - RMApp app1 = rm.submitApp(3072, "app1", "user1", null, "b"); + RMApp app1 = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(3072, rm) + .withName("app1") + .withUser("user1") + .withAcls(null) + .withQueue("b") + .build()); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); WebResource r = resource().path(RMWSConsts.RM_WEB_SERVICE_PATH) @@ -545,7 +583,13 @@ public void testAppGroupByDiagnostics() throws Exception { MockNM nm4 = rm.registerNode("127.0.0.4:1234", 2 * 1024); try { - RMApp app1 = rm.submitApp(3072, "app1", "user1", null, "b"); + RMApp app1 = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(3072, rm) + .withName("app1") + .withUser("user1") + .withAcls(null) + .withQueue("b") + .build()); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); WebResource r = resource().path(RMWSConsts.RM_WEB_SERVICE_PATH)