diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java index ad00bd651bd..4957d97c558 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java @@ -129,6 +129,7 @@ import org.apache.hadoop.yarn.server.api.records.NodeAction; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore; @@ -188,7 +189,7 @@ public void testSimple() throws Exception { rm.start(); // Submit the application - RMApp app = rm.submitApp(1024); + RMApp app = MockRMAppSubmitter.submitWithMemory(1024, rm); rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); @@ -276,7 +277,7 @@ public void testMapNodeLocality() throws Exception { rm.start(); // Submit the application - RMApp app = rm.submitApp(1024); + RMApp app = MockRMAppSubmitter.submitWithMemory(1024, rm); rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); @@ -357,7 +358,7 @@ public void testResource() throws Exception { rm.start(); // Submit the application - RMApp app = rm.submitApp(1024); + RMApp app = MockRMAppSubmitter.submitWithMemory(1024, rm); rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); @@ -421,7 +422,7 @@ public void testReducerRampdownDiagnostics() throws Exception { conf.setFloat(MRJobConfig.COMPLETED_MAPS_FOR_REDUCE_SLOWSTART, 0.0f); final MyResourceManager rm = new MyResourceManager(conf); rm.start(); - final RMApp app = rm.submitApp(1024); + final RMApp app = MockRMAppSubmitter.submitWithMemory(1024, rm); rm.drainEvents(); final String host = "host1"; @@ -477,7 +478,7 @@ public void testPreemptReducers() throws Exception { rm.start(); // Submit the application - RMApp app = rm.submitApp(1024); + RMApp app = MockRMAppSubmitter.submitWithMemory(1024, rm); rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); @@ -529,7 +530,7 @@ public void testNonAggressivelyPreemptReducers() throws Exception { rm.start(); // Submit the application - RMApp app = rm.submitApp(1024); + RMApp app = MockRMAppSubmitter.submitWithMemory(1024, rm); rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); @@ -593,7 +594,7 @@ public void testUnconditionalPreemptReducers() throws Exception { rm.getMyFifoScheduler().forceResourceLimit(Resource.newInstance(8192, 8)); // Submit the application - RMApp app = rm.submitApp(1024); + RMApp app = MockRMAppSubmitter.submitWithMemory(1024, rm); rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); @@ -647,7 +648,7 @@ public void testExcessReduceContainerAssign() throws Exception { conf.setFloat(MRJobConfig.COMPLETED_MAPS_FOR_REDUCE_SLOWSTART, 0.0f); final MyResourceManager2 rm = new MyResourceManager2(conf); rm.start(); - final RMApp app = rm.submitApp(2048); + final RMApp app = MockRMAppSubmitter.submitWithMemory(2048, rm); rm.drainEvents(); final String host = "host1"; final MockNM nm = rm.registerNode(String.format("%s:1234", host), 4096); @@ -878,7 +879,7 @@ public void testMapReduceScheduling() throws Exception { rm.start(); // Submit the application - RMApp app = rm.submitApp(1024); + RMApp app = MockRMAppSubmitter.submitWithMemory(1024, rm); rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); @@ -1010,7 +1011,7 @@ public void testReportedAppProgress() throws Exception { .getDispatcher(); // Submit the application - RMApp rmApp = rm.submitApp(1024); + RMApp rmApp = MockRMAppSubmitter.submitWithMemory(1024, rm); rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 21504); @@ -1162,7 +1163,7 @@ public void testReportedAppProgressWithOnlyMaps() throws Exception { .getDispatcher(); // Submit the application - RMApp rmApp = rm.submitApp(1024); + RMApp rmApp = MockRMAppSubmitter.submitWithMemory(1024, rm); rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 11264); @@ -1255,7 +1256,7 @@ public void testUpdatedNodes() throws Exception { rm.start(); // Submit the application - RMApp app = rm.submitApp(1024); + RMApp app = MockRMAppSubmitter.submitWithMemory(1024, rm); rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); amNodeManager.nodeHeartbeat(true); @@ -1350,7 +1351,7 @@ public void testBlackListedNodes() throws Exception { rm.start(); // Submit the application - RMApp app = rm.submitApp(1024); + RMApp app = MockRMAppSubmitter.submitWithMemory(1024, rm); rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); @@ -1459,7 +1460,7 @@ public void testIgnoreBlacklisting() throws Exception { rm.start(); // Submit the application - RMApp app = rm.submitApp(1024); + RMApp app = MockRMAppSubmitter.submitWithMemory(1024, rm); rm.drainEvents(); MockNM[] nodeManagers = new MockNM[10]; @@ -1648,7 +1649,7 @@ public void testBlackListedNodesWithSchedulingToThatNode() throws Exception { rm.start(); // Submit the application - RMApp app = rm.submitApp(1024); + RMApp app = MockRMAppSubmitter.submitWithMemory(1024, rm); rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); @@ -2321,7 +2322,7 @@ public void testCompletedTasksRecalculateSchedule() throws Exception { rm.start(); // Submit the application - RMApp app = rm.submitApp(1024); + RMApp app = MockRMAppSubmitter.submitWithMemory(1024, rm); rm.drainEvents(); // Make a node to register so as to launch the AM. @@ -2473,7 +2474,7 @@ public void testUnregistrationOnlyIfRegistered() throws Exception { rm.start(); // Submit the application - RMApp rmApp = rm.submitApp(1024); + RMApp rmApp = MockRMAppSubmitter.submitWithMemory(1024, rm); rm.drainEvents(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 11264); @@ -2541,7 +2542,7 @@ public void testRMContainerAllocatorResendsRequestsOnRMRestart() rm1.start(); // Submit the application - RMApp app = rm1.submitApp(1024); + RMApp app = MockRMAppSubmitter.submitWithMemory(1024, rm1); rm1.drainEvents(); MockNM nm1 = new MockNM("h1:1234", 15120, rm1.getResourceTrackerService()); @@ -2794,7 +2795,7 @@ public void testRMUnavailable() MyResourceManager rm1 = new MyResourceManager(conf); rm1.start(); - RMApp app = rm1.submitApp(1024); + RMApp app = MockRMAppSubmitter.submitWithMemory(1024, rm1); rm1.drainEvents(); MockNM nm1 = new MockNM("h1:1234", 15120, rm1.getResourceTrackerService()); @@ -2846,7 +2847,7 @@ public void testAMRMTokenUpdate() throws Exception { rm.getRMContext().getAMRMTokenSecretManager(); // Submit the application - RMApp app = rm.submitApp(1024); + RMApp app = MockRMAppSubmitter.submitWithMemory(1024, rm); rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); @@ -3118,7 +3119,7 @@ public void testAttemptNotFoundCausesRMCommunicatorException() rm.start(); // Submit the application - RMApp app = rm.submitApp(1024); + RMApp app = MockRMAppSubmitter.submitWithMemory(1024, rm); rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); @@ -3152,7 +3153,7 @@ public void testUpdateAskOnRampDownAllReduces() throws Exception { rm.start(); // Submit the application - RMApp app = rm.submitApp(1024); + RMApp app = MockRMAppSubmitter.submitWithMemory(1024, rm); rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 1260); @@ -3323,7 +3324,7 @@ public void testAvoidAskMoreReducersWhenReducerPreemptionIsRequired() rm.start(); // Submit the application - RMApp app = rm.submitApp(1024); + RMApp app = MockRMAppSubmitter.submitWithMemory(1024, rm); rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 1260); @@ -3457,7 +3458,7 @@ public void testExcludeSchedReducesFromHeadroom() throws Exception { rm.start(); // Submit the application - RMApp app = rm.submitApp(1024); + RMApp app = MockRMAppSubmitter.submitWithMemory(1024, rm); rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 1260); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java index 51048660ef2..4adca73a306 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java @@ -58,6 +58,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore; @@ -127,7 +128,7 @@ public void testAMRMClientResendsRequestsOnRMRestart() throws Exception { rm1.start(); // Submit the application - RMApp app = rm1.submitApp(1024); + RMApp app = MockRMAppSubmitter.submitWithMemory(1024, rm1); rm1.drainEvents(); MockNM nm1 = new MockNM("h1:1234", 15120, rm1.getResourceTrackerService()); @@ -356,7 +357,7 @@ public void testAMRMClientForUnregisterAMOnRMRestart() throws Exception { rm1.start(); // Submit the application - RMApp app = rm1.submitApp(1024); + RMApp app = MockRMAppSubmitter.submitWithMemory(1024, rm1); rm1.drainEvents(); MockNM nm1 = new MockNM("h1:1234", 15120, rm1.getResourceTrackerService()); @@ -433,7 +434,7 @@ public void testAMRMClientOnAMRMTokenRollOverOnRMRestart() throws Exception { rm1.start(); Long startTime = System.currentTimeMillis(); // Submit the application - RMApp app = rm1.submitApp(1024); + RMApp app = MockRMAppSubmitter.submitWithMemory(1024, rm1); rm1.drainEvents(); MockNM nm1 = new MockNM("h1:1234", 15120, rm1.getResourceTrackerService()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java index 17e43cacda7..458ae45f92a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java @@ -68,6 +68,8 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.MiniYARNCluster; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.ParameterizedSchedulerTestBase; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; @@ -367,7 +369,7 @@ public void testSubmitIncorrectQueueToCapacityScheduler() throws IOException { public void testKillApplication() throws Exception { MockRM rm = new MockRM(); rm.start(); - RMApp app = rm.submitApp(2000); + RMApp app = MockRMAppSubmitter.submitWithMemory(2000, rm); @SuppressWarnings("resource") final YarnClient client = new MockYarnClient(); @@ -384,11 +386,19 @@ public void testApplicationType() throws Exception { GenericTestUtils.setRootLogLevel(Level.DEBUG); MockRM rm = new MockRM(); rm.start(); - RMApp app = rm.submitApp(2000); + RMApp app = MockRMAppSubmitter.submitWithMemory(2000, rm); RMApp app1 = - rm.submitApp(200, "name", "user", - new HashMap<>(), false, "default", -1, - null, "MAPREDUCE"); + MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm) + .withName("name") + .withUser("user") + .withAcls(new HashMap<>()) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(-1) + .withCredentials(null) + .withAppType("MAPREDUCE") + .build()); Assert.assertEquals("YARN", app.getApplicationType()); Assert.assertEquals("MAPREDUCE", app1.getApplicationType()); rm.stop(); @@ -400,9 +410,17 @@ public void testApplicationTypeLimit() throws Exception { MockRM rm = new MockRM(); rm.start(); RMApp app1 = - rm.submitApp(200, "name", "user", - new HashMap<>(), false, "default", -1, - null, "MAPREDUCE-LENGTH-IS-20"); + MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm) + .withName("name") + .withUser("user") + .withAcls(new HashMap<>()) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(-1) + .withCredentials(null) + .withAppType("MAPREDUCE-LENGTH-IS-20") + .build()); Assert.assertEquals("MAPREDUCE-LENGTH-IS-", app1.getApplicationType()); rm.stop(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterServiceTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterServiceTestBase.java index 7fc2a53cc19..161ef87bff2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterServiceTestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterServiceTestBase.java @@ -131,7 +131,7 @@ public void testRMIdentifierOnContainerAllocation() throws Exception { MockNM nm1 = rm.registerNode(DEFAULT_HOST + ":" + DEFAULT_PORT, 6 * GB); // Submit an application - RMApp app1 = rm.submitApp(2048); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(2048, rm); // kick the scheduling nm1.nodeHeartbeat(true); @@ -172,7 +172,7 @@ public void testAllocateResponseIdOverflow() throws Exception { MockNM nm1 = rm.registerNode(DEFAULT_HOST + ":" + DEFAULT_PORT, 6 * GB); // Submit an application - RMApp app1 = rm.submitApp(2048); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(2048, rm); // kick off the scheduling nm1.nodeHeartbeat(true); @@ -206,7 +206,7 @@ public void testInvalidContainerReleaseRequest() throws Exception { MockNM nm1 = rm.registerNode(DEFAULT_HOST + ":" + DEFAULT_PORT, 6 * GB); // Submit an application - RMApp app1 = rm.submitApp(1024); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(1024, rm); // kick the scheduling nm1.nodeHeartbeat(true); @@ -227,7 +227,7 @@ public void testInvalidContainerReleaseRequest() throws Exception { Assert.assertTrue(alloc1Response.getAllocatedContainers().size() > 0); - RMApp app2 = rm.submitApp(1024); + RMApp app2 = MockRMAppSubmitter.submitWithMemory(1024, rm); nm1.nodeHeartbeat(true); RMAppAttempt attempt2 = app2.getCurrentAppAttempt(); @@ -261,7 +261,7 @@ public void testProgressFilter() throws Exception{ MockNM nm1 = rm.registerNode(DEFAULT_HOST + ":" + DEFAULT_PORT, 6 * GB); // Submit an application - RMApp app1 = rm.submitApp(2048); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(2048, rm); nm1.nodeHeartbeat(true); RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); @@ -326,7 +326,7 @@ public void testFinishApplicationMasterBeforeRegistering() throws Exception { // Register node1 MockNM nm1 = rm.registerNode(DEFAULT_HOST + ":" + DEFAULT_PORT, 6 * GB); // Submit an application - RMApp app1 = rm.submitApp(2048); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(2048, rm); MockAM am1 = MockRM.launchAM(app1, rm, nm1); FinishApplicationMasterRequest req = FinishApplicationMasterRequest.newInstance( @@ -395,7 +395,7 @@ public void testResourceTypes() throws Exception { MockRM rm = new MockRM(entry.getKey()); rm.start(); MockNM nm1 = rm.registerNode(DEFAULT_HOST + ":" + DEFAULT_PORT, 6 * GB); - RMApp app1 = rm.submitApp(2048); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(2048, rm); //Wait to make sure the attempt has the right state //TODO explore a better way than sleeping for a while (YARN-4929) Thread.sleep(1000); @@ -419,7 +419,7 @@ public void testAllocateAfterUnregister() throws Exception { MockNM nm1 = rm.registerNode(DEFAULT_HOST + ":" + DEFAULT_PORT, 6 * GB); // Submit an application - RMApp app1 = rm.submitApp(2048); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(2048, rm); nm1.nodeHeartbeat(true); RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); @@ -450,7 +450,7 @@ public void testUpdateTrackingUrl() throws Exception { // Register node1 MockNM nm1 = rm.registerNode(DEFAULT_HOST + ":" + DEFAULT_PORT, 6 * GB); - RMApp app1 = rm.submitApp(2048); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(2048, rm); nm1.nodeHeartbeat(true); RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); @@ -496,7 +496,15 @@ public void testValidateRequestCapacityAgainstMinMaxAllocation() .createResource(DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, null)); - RMApp app1 = rm.submitApp(GB, "app", "user", null, getDefaultQueueName()); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue(getDefaultQueueName()) + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); // Now request resource, memory > allowed @@ -553,7 +561,15 @@ public void testRequestCapacityMinMaxAllocationForResourceTypes() DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, ImmutableMap.of(CUSTOM_RES, 4))); - RMApp app1 = rm.submitApp(GB, "app", "user", null, getDefaultQueueName()); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue(getDefaultQueueName()) + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); Assert.assertEquals(Resource.newInstance(GB, 1), diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java index 408cb187619..d8c6fe0b794 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java @@ -509,401 +509,6 @@ public GetNewApplicationResponse getNewAppId() throws Exception { .newRecord(GetNewApplicationRequest.class)); } - public RMApp submitApp(int masterMemory) throws Exception { - return submitApp(masterMemory, false); - } - - public RMApp submitApp(int masterMemory, String queue) throws Exception { - return submitApp(masterMemory, "", - UserGroupInformation.getCurrentUser().getShortUserName(), null, false, - queue, super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), - null); - } - - public RMApp submitApp(int masterMemory, Set appTags) - throws Exception { - Resource resource = Resource.newInstance(masterMemory, 0); - ResourceRequest amResourceRequest = ResourceRequest.newInstance( - Priority.newInstance(0), ResourceRequest.ANY, resource, 1); - return submitApp(Collections.singletonList(amResourceRequest), "", - UserGroupInformation.getCurrentUser().getShortUserName(), null, false, - null, super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, true, - false, false, null, 0, null, true, Priority.newInstance(0), null, - null, null, appTags); - } - - public RMApp submitApp(int masterMemory, Priority priority) throws Exception { - Resource resource = Resource.newInstance(masterMemory, 0); - return submitApp(resource, "", UserGroupInformation.getCurrentUser() - .getShortUserName(), null, false, null, - super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, true, - false, false, null, 0, null, true, priority); - } - - public RMApp submitApp(int masterMemory, Priority priority, - Credentials cred, ByteBuffer tokensConf) throws Exception { - Resource resource = Resource.newInstance(masterMemory, 0); - return submitApp(resource, "", UserGroupInformation.getCurrentUser() - .getShortUserName(), null, false, null, - super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), cred, null, true, - false, false, null, 0, null, true, priority, null, null, - tokensConf); - } - - public RMApp submitApp(int masterMemory, boolean unmanaged) - throws Exception { - return submitApp(masterMemory, "", UserGroupInformation.getCurrentUser() - .getShortUserName(), unmanaged); - } - - // client - public RMApp submitApp(int masterMemory, String name, String user) throws Exception { - return submitApp(masterMemory, name, user, false); - } - - public RMApp submitApp(int masterMemory, String name, String user, - boolean unmanaged) - throws Exception { - return submitApp(masterMemory, name, user, null, unmanaged, null, - super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null); - } - - public RMApp submitApp(int masterMemory, String name, String user, - Map acls) throws Exception { - return submitApp(masterMemory, name, user, acls, false, null, - super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null); - } - - public RMApp submitApp(int masterMemory, String name, String user, - Map acls, String queue) throws Exception { - return submitApp(masterMemory, name, user, acls, false, queue, - super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null); - } - - public RMApp submitApp(int masterMemory, String name, String user, - Map acls, String queue, String amLabel) - throws Exception { - Resource resource = Records.newRecord(Resource.class); - resource.setMemorySize(masterMemory); - Priority priority = Priority.newInstance(0); - return submitApp(resource, name, user, acls, false, queue, - super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, true, false, - false, null, 0, null, true, priority, amLabel, null, null); - } - - public RMApp submitApp(Resource resource, String name, String user, - Map acls, String queue) throws Exception { - return submitApp(resource, name, user, acls, false, queue, - super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, - true, false, false, null, 0, null, true, null); - } - - public RMApp submitApp(Resource resource, String name, String user, - Map acls, boolean unManaged, String queue) - throws Exception { - return submitApp(resource, name, user, acls, unManaged, queue, - super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, true, - false, false, null, 0, null, true, null); - } - - public RMApp submitApp(int masterMemory, String name, String user, - Map acls, String queue, - boolean waitForAccepted) throws Exception { - return submitApp(masterMemory, name, user, acls, false, queue, - super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, - waitForAccepted); - } - - public RMApp submitApp(int masterMemory, String name, String user, - Map acls, boolean unmanaged, String queue, - int maxAppAttempts, Credentials ts) throws Exception { - return submitApp(masterMemory, name, user, acls, unmanaged, queue, - maxAppAttempts, ts, null); - } - - public RMApp submitApp(int masterMemory, String name, String user, - Map acls, boolean unmanaged, String queue, - int maxAppAttempts, Credentials ts, String appType) throws Exception { - return submitApp(masterMemory, name, user, acls, unmanaged, queue, - maxAppAttempts, ts, appType, true); - } - - public RMApp submitApp(int masterMemory, String name, String user, - Map acls, boolean unmanaged, String queue, - int maxAppAttempts, Credentials ts, String appType, - boolean waitForAccepted) - throws Exception { - return submitApp(masterMemory, name, user, acls, unmanaged, queue, - maxAppAttempts, ts, appType, waitForAccepted, false); - } - - public RMApp submitApp(int masterMemory, String name, String user, - Map acls, boolean unmanaged, String queue, - int maxAppAttempts, Credentials ts, String appType, - boolean waitForAccepted, boolean keepContainers) throws Exception { - Resource resource = Records.newRecord(Resource.class); - resource.setMemorySize(masterMemory); - return submitApp(resource, name, user, acls, unmanaged, queue, - maxAppAttempts, ts, appType, waitForAccepted, keepContainers, - false, null, 0, null, true, Priority.newInstance(0)); - } - - public RMApp submitApp(int masterMemory, long attemptFailuresValidityInterval, - boolean keepContainers) throws Exception { - Resource resource = Records.newRecord(Resource.class); - resource.setMemorySize(masterMemory); - Priority priority = Priority.newInstance(0); - return submitApp(resource, "", UserGroupInformation.getCurrentUser() - .getShortUserName(), null, false, null, - super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, true, keepContainers, - false, null, attemptFailuresValidityInterval, null, true, priority); - } - - public RMApp submitApp(int masterMemory, String name, String user, - Map acls, boolean unmanaged, String queue, - int maxAppAttempts, Credentials ts, String appType, - boolean waitForAccepted, boolean keepContainers, boolean isAppIdProvided, - ApplicationId applicationId) throws Exception { - Resource resource = Records.newRecord(Resource.class); - resource.setMemorySize(masterMemory); - Priority priority = Priority.newInstance(0); - return submitApp(resource, name, user, acls, unmanaged, queue, - maxAppAttempts, ts, appType, waitForAccepted, keepContainers, - isAppIdProvided, applicationId, 0, null, true, priority); - } - - public RMApp submitApp(int masterMemory, - LogAggregationContext logAggregationContext) throws Exception { - Resource resource = Records.newRecord(Resource.class); - resource.setMemorySize(masterMemory); - Priority priority = Priority.newInstance(0); - return submitApp(resource, "", UserGroupInformation.getCurrentUser() - .getShortUserName(), null, false, null, - super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, true, false, - false, null, 0, logAggregationContext, true, priority); - } - - public RMApp submitApp(Resource capability, String name, String user, - Map acls, boolean unmanaged, String queue, - int maxAppAttempts, Credentials ts, String appType, - boolean waitForAccepted, boolean keepContainers, boolean isAppIdProvided, - ApplicationId applicationId, long attemptFailuresValidityInterval, - LogAggregationContext logAggregationContext, - boolean cancelTokensWhenComplete, Priority priority) throws Exception { - return submitApp(capability, name, user, acls, unmanaged, queue, - maxAppAttempts, ts, appType, waitForAccepted, keepContainers, - isAppIdProvided, applicationId, attemptFailuresValidityInterval, - logAggregationContext, cancelTokensWhenComplete, priority, "", null, - null); - } - - public RMApp submitApp(Credentials cred, ByteBuffer tokensConf) - throws Exception { - return submitApp(Resource.newInstance(200, 1), "app1", "user", null, false, - null, super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), cred, null, true, - false, false, null, 0, null, true, Priority.newInstance(0), null, null, - tokensConf); - } - - public RMApp submitApp(List amResourceRequests) - throws Exception { - return submitApp(amResourceRequests, "app1", - "user", null, false, null, - super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, true, - false, false, null, 0, null, true, - amResourceRequests.get(0).getPriority(), - amResourceRequests.get(0).getNodeLabelExpression(), null, null); - } - - public RMApp submitApp(List amResourceRequests, - String appNodeLabel) throws Exception { - return submitApp(amResourceRequests, "app1", "user", null, false, null, - super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, true, - false, false, null, 0, null, true, - amResourceRequests.get(0).getPriority(), - amResourceRequests.get(0).getNodeLabelExpression(), null, null, null, - appNodeLabel); - } - - public RMApp submitApp(Resource capability, String name, String user, - Map acls, boolean unmanaged, String queue, - int maxAppAttempts, Credentials ts, String appType, - boolean waitForAccepted, boolean keepContainers, boolean isAppIdProvided, - ApplicationId applicationId, long attemptFailuresValidityInterval, - LogAggregationContext logAggregationContext, - boolean cancelTokensWhenComplete, Priority priority, String amLabel, - Map applicationTimeouts, - ByteBuffer tokensConf) - throws Exception { - priority = (priority == null) ? Priority.newInstance(0) : priority; - ResourceRequest amResourceRequest = ResourceRequest.newInstance( - priority, ResourceRequest.ANY, capability, 1); - if (amLabel != null && !amLabel.isEmpty()) { - amResourceRequest.setNodeLabelExpression(amLabel.trim()); - } - return submitApp(Collections.singletonList(amResourceRequest), name, user, - acls, unmanaged, queue, maxAppAttempts, ts, appType, waitForAccepted, - keepContainers, isAppIdProvided, applicationId, - attemptFailuresValidityInterval, logAggregationContext, - cancelTokensWhenComplete, priority, amLabel, applicationTimeouts, - tokensConf); - } - - public RMApp submitApp(List amResourceRequests, String name, - String user, Map acls, boolean unmanaged, - String queue, int maxAppAttempts, Credentials ts, String appType, - boolean waitForAccepted, boolean keepContainers, boolean isAppIdProvided, - ApplicationId applicationId, long attemptFailuresValidityInterval, - LogAggregationContext logAggregationContext, - boolean cancelTokensWhenComplete, Priority priority, String amLabel, - Map applicationTimeouts, - ByteBuffer tokensConf) throws Exception { - return submitApp(amResourceRequests, name, user, acls, unmanaged, queue, - maxAppAttempts, ts, appType, waitForAccepted, keepContainers, - isAppIdProvided, applicationId, attemptFailuresValidityInterval, - logAggregationContext, cancelTokensWhenComplete, priority, amLabel, - applicationTimeouts, tokensConf, null); - } - - public RMApp submitApp(List amResourceRequests, String name, - String user, Map acls, boolean unmanaged, - String queue, int maxAppAttempts, Credentials ts, String appType, - boolean waitForAccepted, boolean keepContainers, boolean isAppIdProvided, - ApplicationId applicationId, long attemptFailuresValidityInterval, - LogAggregationContext logAggregationContext, - boolean cancelTokensWhenComplete, Priority priority, String amLabel, - Map applicationTimeouts, - ByteBuffer tokensConf, Set applicationTags) throws Exception { - return submitApp(amResourceRequests, name, user, acls, unmanaged, queue, - maxAppAttempts, ts, appType, waitForAccepted, keepContainers, - isAppIdProvided, applicationId, attemptFailuresValidityInterval, - logAggregationContext, cancelTokensWhenComplete, priority, amLabel, - applicationTimeouts, tokensConf, applicationTags, null); - } - - public RMApp submitApp(List amResourceRequests, String name, - String user, Map acls, boolean unmanaged, - String queue, int maxAppAttempts, Credentials ts, String appType, - boolean waitForAccepted, boolean keepContainers, boolean isAppIdProvided, - ApplicationId applicationId, long attemptFailuresValidityInterval, - LogAggregationContext logAggregationContext, - boolean cancelTokensWhenComplete, Priority priority, String amLabel, - Map applicationTimeouts, - ByteBuffer tokensConf, Set applicationTags, String appNodeLabel) - throws Exception { - ApplicationId appId = isAppIdProvided ? applicationId : null; - ApplicationClientProtocol client = getClientRMService(); - if (! isAppIdProvided) { - GetNewApplicationResponse resp = client.getNewApplication(Records - .newRecord(GetNewApplicationRequest.class)); - appId = resp.getApplicationId(); - } - SubmitApplicationRequest req = Records - .newRecord(SubmitApplicationRequest.class); - ApplicationSubmissionContext sub = Records - .newRecord(ApplicationSubmissionContext.class); - sub.setKeepContainersAcrossApplicationAttempts(keepContainers); - sub.setApplicationId(appId); - sub.setApplicationName(name); - sub.setMaxAppAttempts(maxAppAttempts); - if (applicationTags != null) { - sub.setApplicationTags(applicationTags); - } - if (applicationTimeouts != null && applicationTimeouts.size() > 0) { - sub.setApplicationTimeouts(applicationTimeouts); - } - if (unmanaged) { - sub.setUnmanagedAM(true); - } - if (queue != null) { - sub.setQueue(queue); - } - if (priority != null) { - sub.setPriority(priority); - } - if (appNodeLabel != null) { - sub.setNodeLabelExpression(appNodeLabel); - } - sub.setApplicationType(appType); - ContainerLaunchContext clc = Records - .newRecord(ContainerLaunchContext.class); - clc.setApplicationACLs(acls); - if (ts != null && UserGroupInformation.isSecurityEnabled()) { - DataOutputBuffer dob = new DataOutputBuffer(); - ts.writeTokenStorageToStream(dob); - ByteBuffer securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); - clc.setTokens(securityTokens); - clc.setTokensConf(tokensConf); - } - sub.setAMContainerSpec(clc); - sub.setAttemptFailuresValidityInterval(attemptFailuresValidityInterval); - if (logAggregationContext != null) { - sub.setLogAggregationContext(logAggregationContext); - } - sub.setCancelTokensWhenComplete(cancelTokensWhenComplete); - if (amLabel != null && !amLabel.isEmpty()) { - for (ResourceRequest amResourceRequest : amResourceRequests) { - amResourceRequest.setNodeLabelExpression(amLabel.trim()); - } - } - sub.setAMContainerResourceRequests(amResourceRequests); - req.setApplicationSubmissionContext(sub); - UserGroupInformation fakeUser = - UserGroupInformation.createUserForTesting(user, new String[] {"someGroup"}); - PrivilegedExceptionAction action = - new PrivilegedExceptionAction() { - ApplicationClientProtocol client; - SubmitApplicationRequest req; - @Override - public SubmitApplicationResponse run() throws IOException, YarnException { - try { - return client.submitApplication(req); - } catch (YarnException | IOException e) { - e.printStackTrace(); - throw e; - } - } - PrivilegedExceptionAction setClientReq( - ApplicationClientProtocol client, SubmitApplicationRequest req) { - this.client = client; - this.req = req; - return this; - } - }.setClientReq(client, req); - fakeUser.doAs(action); - // make sure app is immediately available after submit - if (waitForAccepted) { - waitForState(appId, RMAppState.ACCEPTED); - } - RMApp rmApp = getRMContext().getRMApps().get(appId); - - // unmanaged AM won't go to RMAppAttemptState.SCHEDULED. - if (waitForAccepted && !unmanaged) { - waitForState(rmApp.getCurrentAppAttempt().getAppAttemptId(), - RMAppAttemptState.SCHEDULED); - } - - ((AbstractYarnScheduler)getResourceScheduler()).update(); - - return rmApp; - } - public MockNM unRegisterNode(MockNM nm) throws Exception { nm.unRegisterNode(); drainEventsImplicitly(); @@ -1412,18 +1017,6 @@ public void enableDrainEventsImplicityly() { disableDrainEventsImplicitly = false; } - public RMApp submitApp(int masterMemory, Priority priority, - Map applicationTimeouts) throws Exception { - Resource resource = Resource.newInstance(masterMemory, 0); - return submitApp( - resource, "", UserGroupInformation.getCurrentUser().getShortUserName(), - null, false, null, - super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, true, - false, false, null, 0, null, true, priority, null, applicationTimeouts, - null); - } - @Override protected void serviceInit(Configuration conf) throws Exception { super.serviceInit(conf); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRMAppSubmissionData.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRMAppSubmissionData.java new file mode 100644 index 00000000000..d647444acc2 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRMAppSubmissionData.java @@ -0,0 +1,364 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager; + +import org.apache.hadoop.security.Credentials; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.api.records.ApplicationAccessType; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType; +import org.apache.hadoop.yarn.api.records.LogAggregationContext; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.util.Records; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class MockRMAppSubmissionData { + private final List amResourceRequests; + private final String name; + private final String user; + private final Map acls; + private final boolean unmanaged; + private final String queue; + private final int maxAppAttempts; + private final String appType; + private final boolean waitForAccepted; + private final boolean keepContainers; + private final boolean isAppIdProvided; + private final ApplicationId applicationId; + private final long attemptFailuresValidityInterval; + private final LogAggregationContext logAggregationContext; + private final boolean cancelTokensWhenComplete; + private final Priority priority; + private final String amLabel; + private final Map applicationTimeouts; + private final ByteBuffer tokensConf; + private final Set applicationTags; + private final String appNodeLabel; + private final Credentials credentials; + private final Resource resource; + + public List getAmResourceRequests() { + return amResourceRequests; + } + + public String getName() { + return name; + } + + public String getUser() { + return user; + } + + public Map getAcls() { + return acls; + } + + public boolean isUnmanaged() { + return unmanaged; + } + + public String getQueue() { + return queue; + } + + public int getMaxAppAttempts() { + return maxAppAttempts; + } + + public String getAppType() { + return appType; + } + + public boolean isWaitForAccepted() { + return waitForAccepted; + } + + public boolean isKeepContainers() { + return keepContainers; + } + + public boolean isAppIdProvided() { + return isAppIdProvided; + } + + public ApplicationId getApplicationId() { + return applicationId; + } + + public long getAttemptFailuresValidityInterval() { + return attemptFailuresValidityInterval; + } + + public LogAggregationContext getLogAggregationContext() { + return logAggregationContext; + } + + public boolean isCancelTokensWhenComplete() { + return cancelTokensWhenComplete; + } + + public Priority getPriority() { + return priority; + } + + public String getAmLabel() { + return amLabel; + } + + public Map getApplicationTimeouts() { + return applicationTimeouts; + } + + public ByteBuffer getTokensConf() { + return tokensConf; + } + + public Set getApplicationTags() { + return applicationTags; + } + + public String getAppNodeLabel() { + return appNodeLabel; + } + + public Credentials getCredentials() { + return credentials; + } + + public Resource getResource() { + return resource; + } + + private MockRMAppSubmissionData(Builder builder) { + this.amLabel = builder.amLabel; + this.tokensConf = builder.tokensConf; + this.maxAppAttempts = builder.maxAppAttempts; + this.logAggregationContext = builder.logAggregationContext; + this.queue = builder.queue; + this.amResourceRequests = builder.amResourceRequests; + this.user = builder.user; + this.priority = builder.priority; + this.waitForAccepted = builder.waitForAccepted; + this.keepContainers = builder.keepContainers; + this.name = builder.name; + this.applicationId = builder.applicationId; + this.attemptFailuresValidityInterval = builder.attemptFailuresValidityInterval; + this.acls = builder.acls; + this.appType = builder.appType; + this.appNodeLabel = builder.appNodeLabel; + this.isAppIdProvided = builder.isAppIdProvided; + this.unmanaged = builder.unmanaged; + this.applicationTags = builder.applicationTags; + this.cancelTokensWhenComplete = builder.cancelTokensWhenComplete; + this.applicationTimeouts = builder.applicationTimeouts; + this.credentials = builder.credentials; + this.resource = builder.resource; + } + + public static final class Builder { + private List amResourceRequests; + private String name; + private String user; + private Map acls; + private boolean unmanaged; + private String queue; + private int maxAppAttempts; + private String appType; + private boolean waitForAccepted; + private boolean keepContainers; + private boolean isAppIdProvided; + private ApplicationId applicationId; + private long attemptFailuresValidityInterval; + private LogAggregationContext logAggregationContext; + private boolean cancelTokensWhenComplete; + private Priority priority; + private String amLabel; + private Map applicationTimeouts; + private ByteBuffer tokensConf; + private Set applicationTags; + private String appNodeLabel; + private Credentials credentials; + private Resource resource; + + private Builder() { + } + + public static Builder create() { + return new Builder(); + } + + public static Builder createWithDefaults(long memory, MockRM mockRM) throws IOException { + Resource resource = Records.newRecord(Resource.class); + resource.setMemorySize(memory); + return createWithDefaults(resource, mockRM); + } + + public static Builder createWithDefaults(Resource resource, MockRM mockRM) throws IOException { + int maxAppAttempts = mockRM.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, + YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS); + return MockRMAppSubmissionData.Builder.create() + .withResource(resource) + .withName("") + .withUser(UserGroupInformation + .getCurrentUser().getShortUserName()) + .withAcls(null) + .withUnmanaged(false) + .withQueue(null) + .withMaxAppAttempts(maxAppAttempts) + .withCredentials(null) + .withAppType(null) + .withWaitForAccepted(true) + .withKeepContainers(false) + .withIsAppIdProvided(false) + .withApplicationId(null) + .withAttemptFailuresValidityInterval(0L) + .withLogAggregationContext(null) + .withCancelTokensWhenComplete(true) + .withPriority(Priority.newInstance(0)) + .withAmLabel("") + .withApplicationTimeouts(null) + .withTokensConf(null); + } + + public Builder withAmResourceRequests(List amResourceRequests) { + this.amResourceRequests = amResourceRequests; + return this; + } + + public Builder withName(String name) { + this.name = name; + return this; + } + + public Builder withUser(String user) { + this.user = user; + return this; + } + + public Builder withAcls(Map acls) { + this.acls = acls; + return this; + } + + public Builder withUnmanaged(boolean unmanaged) { + this.unmanaged = unmanaged; + return this; + } + + public Builder withQueue(String queue) { + this.queue = queue; + return this; + } + + public Builder withMaxAppAttempts(int maxAppAttempts) { + this.maxAppAttempts = maxAppAttempts; + return this; + } + + public Builder withAppType(String appType) { + this.appType = appType; + return this; + } + + public Builder withWaitForAccepted(boolean waitForAccepted) { + this.waitForAccepted = waitForAccepted; + return this; + } + + public Builder withKeepContainers(boolean keepContainers) { + this.keepContainers = keepContainers; + return this; + } + + public Builder withIsAppIdProvided(boolean isAppIdProvided) { + this.isAppIdProvided = isAppIdProvided; + return this; + } + + public Builder withApplicationId(ApplicationId applicationId) { + this.applicationId = applicationId; + return this; + } + + public Builder withAttemptFailuresValidityInterval(long attemptFailuresValidityInterval) { + this.attemptFailuresValidityInterval = attemptFailuresValidityInterval; + return this; + } + + public Builder withLogAggregationContext(LogAggregationContext logAggregationContext) { + this.logAggregationContext = logAggregationContext; + return this; + } + + public Builder withCancelTokensWhenComplete(boolean cancelTokensWhenComplete) { + this.cancelTokensWhenComplete = cancelTokensWhenComplete; + return this; + } + + public Builder withPriority(Priority priority) { + this.priority = priority; + return this; + } + + public Builder withAmLabel(String amLabel) { + this.amLabel = amLabel; + return this; + } + + public Builder withApplicationTimeouts(Map applicationTimeouts) { + this.applicationTimeouts = applicationTimeouts; + return this; + } + + public Builder withTokensConf(ByteBuffer tokensConf) { + this.tokensConf = tokensConf; + return this; + } + + public Builder withApplicationTags(Set applicationTags) { + this.applicationTags = applicationTags; + return this; + } + + public Builder withAppNodeLabel(String appNodeLabel) { + this.appNodeLabel = appNodeLabel; + return this; + } + + public Builder withCredentials(Credentials cred) { + this.credentials = cred; + return this; + } + + public Builder withResource(Resource resource) { + this.resource = resource; + return this; + } + + public MockRMAppSubmissionData build() { + return new MockRMAppSubmissionData(this); + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRMAppSubmitter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRMAppSubmitter.java new file mode 100644 index 00000000000..a851aa937a1 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRMAppSubmitter.java @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.security.PrivilegedExceptionAction; +import java.util.Collections; +import java.util.List; + +import org.apache.hadoop.io.DataOutputBuffer; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; +import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler; +import org.apache.hadoop.yarn.util.Records; + +public class MockRMAppSubmitter { + public static RMApp submitWithMemory(long memory, MockRM mockRM) throws Exception { + Resource resource = Records.newRecord(Resource.class); + resource.setMemorySize(memory); + MockRMAppSubmissionData data = MockRMAppSubmissionData.Builder.createWithDefaults(resource, mockRM) + .build(); + return MockRMAppSubmitter.submit(mockRM, data); + } + + public static RMApp submit(MockRM mockRM, MockRMAppSubmissionData data) throws Exception { + ApplicationId appId = data.isAppIdProvided() ? data.getApplicationId() : null; + ApplicationClientProtocol client = mockRM.getClientRMService(); + if (!data.isAppIdProvided()) { + GetNewApplicationResponse resp = client.getNewApplication(Records + .newRecord(GetNewApplicationRequest.class)); + appId = resp.getApplicationId(); + } + SubmitApplicationRequest req = Records + .newRecord(SubmitApplicationRequest.class); + ApplicationSubmissionContext sub = Records + .newRecord(ApplicationSubmissionContext.class); + sub.setKeepContainersAcrossApplicationAttempts(data.isKeepContainers()); + sub.setApplicationId(appId); + sub.setApplicationName(data.getName()); + sub.setMaxAppAttempts(data.getMaxAppAttempts()); + if (data.getApplicationTags() != null) { + sub.setApplicationTags(data.getApplicationTags()); + } + if (data.getApplicationTimeouts() != null && data.getApplicationTimeouts().size() > 0) { + sub.setApplicationTimeouts(data.getApplicationTimeouts()); + } + if (data.isUnmanaged()) { + sub.setUnmanagedAM(true); + } + if (data.getQueue() != null) { + sub.setQueue(data.getQueue()); + } + if (data.getPriority() != null) { + sub.setPriority(data.getPriority()); + } + if (data.getAppNodeLabel() != null) { + sub.setNodeLabelExpression(data.getAppNodeLabel()); + } + sub.setApplicationType(data.getAppType()); + ContainerLaunchContext clc = Records + .newRecord(ContainerLaunchContext.class); + clc.setApplicationACLs(data.getAcls()); + if (data.getCredentials() != null && UserGroupInformation.isSecurityEnabled()) { + DataOutputBuffer dob = new DataOutputBuffer(); + data.getCredentials().writeTokenStorageToStream(dob); + ByteBuffer securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); + clc.setTokens(securityTokens); + clc.setTokensConf(data.getTokensConf()); + } + sub.setAMContainerSpec(clc); + sub.setAttemptFailuresValidityInterval(data.getAttemptFailuresValidityInterval()); + if (data.getLogAggregationContext() != null) { + sub.setLogAggregationContext(data.getLogAggregationContext()); + } + sub.setCancelTokensWhenComplete(data.isCancelTokensWhenComplete()); + + Priority priority = data.getPriority(); + if (priority == null) { + priority = Priority.newInstance(0); + } + + List amResourceRequests = data.getAmResourceRequests(); + if (amResourceRequests == null || !amResourceRequests.isEmpty()) { + ResourceRequest amResReq = ResourceRequest.newInstance( + priority, ResourceRequest.ANY, data.getResource(), 1); + amResourceRequests = Collections.singletonList(amResReq); + } + + if (data.getAmLabel() != null && !data.getAmLabel().isEmpty()) { + for (ResourceRequest amResourceRequest : amResourceRequests) { + amResourceRequest.setNodeLabelExpression(data.getAmLabel().trim()); + } + } + sub.setAMContainerResourceRequests(amResourceRequests); + + req.setApplicationSubmissionContext(sub); + UserGroupInformation fakeUser = + UserGroupInformation.createUserForTesting(data.getUser(), new String[] {"someGroup"}); + PrivilegedExceptionAction action = + new PrivilegedExceptionAction() { + ApplicationClientProtocol client; + SubmitApplicationRequest req; + @Override + public SubmitApplicationResponse run() throws IOException, YarnException { + try { + return client.submitApplication(req); + } catch (YarnException | IOException e) { + e.printStackTrace(); + throw e; + } + } + PrivilegedExceptionAction setClientReq( + ApplicationClientProtocol client, SubmitApplicationRequest req) { + this.client = client; + this.req = req; + return this; + } + }.setClientReq(client, req); + fakeUser.doAs(action); + // make sure app is immediately available after submit + if (data.isWaitForAccepted()) { + mockRM.waitForState(appId, RMAppState.ACCEPTED); + } + RMApp rmApp = mockRM.getRMContext().getRMApps().get(appId); + + // unmanaged AM won't go to RMAppAttemptState.SCHEDULED. + if (data.isWaitForAccepted() && !data.isUnmanaged()) { + mockRM.waitForState(rmApp.getCurrentAppAttempt().getAppAttemptId(), + RMAppAttemptState.SCHEDULED); + } + + ((AbstractYarnScheduler)mockRM.getResourceScheduler()).update(); + + return rmApp; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java index 1acf658b229..fec30f3a19c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java @@ -260,7 +260,13 @@ public void testAuthorizedAccess() throws Exception { Map acls = new HashMap(2); acls.put(ApplicationAccessType.VIEW_APP, "*"); - RMApp app = rm.submitApp(1024, "appname", "appuser", acls); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm) + .withName("appname") + .withUser("appuser") + .withAcls(acls) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); nm1.nodeHeartbeat(true); @@ -318,7 +324,7 @@ public void testUnauthorizedAccess() throws Exception { MockNM nm1 = rm.registerNode("localhost:1234", 5120); - RMApp app = rm.submitApp(1024); + RMApp app = MockRMAppSubmitter.submitWithMemory(1024, rm); nm1.nodeHeartbeat(true); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationCleanup.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationCleanup.java index c12ae3385d0..71768c28a0d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationCleanup.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationCleanup.java @@ -84,7 +84,7 @@ public void testAppCleanup() throws Exception { MockNM nm1 = rm.registerNode("127.0.0.1:1234", 5000); - RMApp app = rm.submitApp(2000); + RMApp app = MockRMAppSubmitter.submitWithMemory(2000, rm); //kick the scheduling nm1.nodeHeartbeat(true); @@ -163,7 +163,7 @@ public void testContainerCleanup() throws Exception { MockNM nm1 = rm.registerNode("127.0.0.1:1234", 5000); - RMApp app = rm.submitApp(2000); + RMApp app = MockRMAppSubmitter.submitWithMemory(2000, rm); //kick the scheduling nm1.nodeHeartbeat(true); @@ -300,7 +300,7 @@ public void testAppCleanupWhenRMRestartedAfterAppFinished() throws Exception { nm1.registerNode(); // create app and launch the AM - RMApp app0 = rm1.submitApp(200); + RMApp app0 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am0 = launchAM(app0, rm1, nm1); nm1.nodeHeartbeat(am0.getApplicationAttemptId(), 1, ContainerState.COMPLETE); rm1.waitForState(app0.getApplicationId(), RMAppState.FAILED); @@ -337,7 +337,7 @@ public void testAppCleanupWhenRMRestartedBeforeAppFinished() throws Exception { nm2.registerNode(); // create app and launch the AM - RMApp app0 = rm1.submitApp(200); + RMApp app0 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am0 = launchAM(app0, rm1, nm1); // alloc another container on nm2 @@ -391,7 +391,7 @@ public void testContainerCleanupWhenRMRestartedAppNotRegistered() throws nm1.registerNode(); // create app and launch the AM - RMApp app0 = rm1.submitApp(200); + RMApp app0 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am0 = launchAM(app0, rm1, nm1); nm1.nodeHeartbeat(am0.getApplicationAttemptId(), 1, ContainerState.RUNNING); rm1.waitForState(app0.getApplicationId(), RMAppState.RUNNING); @@ -427,7 +427,7 @@ public void testAppCleanupWhenNMReconnects() throws Exception { nm1.registerNode(); // create app and launch the AM - RMApp app0 = rm1.submitApp(200); + RMApp app0 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am0 = launchAM(app0, rm1, nm1); nm1.nodeHeartbeat(am0.getApplicationAttemptId(), 1, ContainerState.COMPLETE); rm1.waitForState(app0.getApplicationId(), RMAppState.FAILED); @@ -467,7 +467,7 @@ public void testProcessingNMContainerStatusesOnNMRestart() throws Exception { new MockNM("127.0.0.1:1234", nmMemory, rm1.getResourceTrackerService()); nm1.registerNode(); - RMApp app0 = rm1.submitApp(amMemory); + RMApp app0 = MockRMAppSubmitter.submitWithMemory(amMemory, rm1); MockAM am0 = MockRM.launchAndRegisterAM(app0, rm1, nm1); // 2. AM sends ResourceRequest for 1 container with memory 2048MB. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java index 03ccd764b6c..fb296459334 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java @@ -224,7 +224,7 @@ public void testAMLaunchAndCleanup() throws Exception { rm.start(); MockNM nm1 = rm.registerNode("127.0.0.1:1234", 5120); - RMApp app = rm.submitApp(2000); + RMApp app = MockRMAppSubmitter.submitWithMemory(2000, rm); // kick the scheduling nm1.nodeHeartbeat(true); @@ -283,7 +283,7 @@ public void testAMCleanupBeforeLaunch() throws Exception { MockRM rm = new MockRM(); rm.start(); MockNM nm1 = rm.registerNode("127.0.0.1:1234", 5120); - RMApp app = rm.submitApp(2000); + RMApp app = MockRMAppSubmitter.submitWithMemory(2000, rm); // kick the scheduling nm1.nodeHeartbeat(true); RMAppAttempt attempt = app.getCurrentAppAttempt(); @@ -359,7 +359,7 @@ protected YarnRPC getYarnRPC() { rm.start(); MockNM nm1 = rm.registerNode("127.0.0.1:1234", 5120); - RMApp app = rm.submitApp(2000); + RMApp app = MockRMAppSubmitter.submitWithMemory(2000, rm); // kick the scheduling nm1.nodeHeartbeat(true); @@ -380,7 +380,7 @@ public void testallocateBeforeAMRegistration() throws Exception { MockRM rm = new MockRM(); rm.start(); MockNM nm1 = rm.registerNode("h1:1234", 5000); - RMApp app = rm.submitApp(2000); + RMApp app = MockRMAppSubmitter.submitWithMemory(2000, rm); // kick the scheduling nm1.nodeHeartbeat(true); RMAppAttempt attempt = app.getCurrentAppAttempt(); @@ -453,7 +453,7 @@ private void testSetupTokens(boolean https, YarnConfiguration conf) MockRM rm = new MockRM(conf); rm.start(); MockNM nm1 = rm.registerNode("h1:1234", 5000); - RMApp app = rm.submitApp(2000); + RMApp app = MockRMAppSubmitter.submitWithMemory(2000, rm); /// kick the scheduling nm1.nodeHeartbeat(true); RMAppAttempt attempt = app.getCurrentAppAttempt(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterServiceCapacity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterServiceCapacity.java index 8d04e5061c2..e99427320a7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterServiceCapacity.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterServiceCapacity.java @@ -105,7 +105,7 @@ public void testInvalidIncreaseDecreaseRequest() throws Exception { MockNM nm1 = rm.registerNode(DEFAULT_HOST + ":" + DEFAULT_PORT, 6 * GB); // Submit an application - RMApp app1 = rm.submitApp(1024); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(1024, rm); // kick the scheduling nm1.nodeHeartbeat(true); @@ -177,7 +177,10 @@ public void testPriorityInAllocatedResponse() throws Exception { // Submit an application Priority appPriority1 = Priority.newInstance(5); - RMApp app1 = rm.submitApp(2048, appPriority1); + MockRMAppSubmissionData data = MockRMAppSubmissionData.Builder.createWithDefaults(2048, rm) + .withPriority(appPriority1) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); nm1.nodeHeartbeat(true); RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterServiceFair.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterServiceFair.java index 3ff3bcd1224..6b57ca189ab 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterServiceFair.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterServiceFair.java @@ -97,7 +97,15 @@ public void testRequestCapacityMinMaxAllocationWithDifferentUnits() ImmutableMap. builder() .put(CUSTOM_RES, "5G").build())); - RMApp app1 = rm.submitApp(GB, "app", "user", null, DEFAULT_QUEUE); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue(DEFAULT_QUEUE) + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); // Now request res_1, 500M < 5G so it should be allowed diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterServiceInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterServiceInterceptor.java index 66fda4521a9..13bb2d03e52 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterServiceInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterServiceInterceptor.java @@ -171,7 +171,7 @@ public void testApplicationMasterInterceptor() throws Exception { MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * GB); // Submit an application - RMApp app1 = rm.submitApp(2048); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(2048, rm); // kick the scheduling nm1.nodeHeartbeat(true); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestCapacitySchedulerMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestCapacitySchedulerMetrics.java index eaa966afac3..fc8a21dd2ac 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestCapacitySchedulerMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestCapacitySchedulerMetrics.java @@ -78,8 +78,18 @@ public RMNodeLabelsManager createNodeLabelManager() { Assert.assertEquals(0, csMetrics.getNumOfAllocates()); Assert.assertEquals(0, csMetrics.getNumOfCommitSuccess()); - RMApp rmApp = rm.submitApp(1024, "app", "user", null, false, - "default", 1, null, null, false); + RMApp rmApp = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(1) + .withCredentials(null) + .withAppType(null) + .withWaitForAccepted(false) + .build()); MockAM am = MockRM.launchAMWhenAsyncSchedulingEnabled(rmApp, rm); am.registerAppAttempt(); am.allocate("*", 1024, 1, new ArrayList<>()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index 0784d1e5000..6d7f4f21bde 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -597,8 +597,12 @@ public void testForceKillApplication() throws Exception { GetApplicationsRequest getRequest = GetApplicationsRequest.newInstance( EnumSet.of(YarnApplicationState.KILLED)); - RMApp app1 = rm.submitApp(1024); - RMApp app2 = rm.submitApp(1024, true); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(1024, rm); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm) + .withUnmanaged(true) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm, data); assertEquals("Incorrect number of apps in the RM", 0, rmService.getApplications(getRequest).getApplicationList().size()); @@ -2255,7 +2259,10 @@ public void testUpdatePriorityAndKillAppWithZeroClusterResource() MockRM rm = new MockRM(conf); rm.init(conf); rm.start(); - RMApp app1 = rm.submitApp(1024, Priority.newInstance(appPriority)); + MockRMAppSubmissionData data = MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm) + .withPriority(Priority.newInstance(appPriority)) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); ClientRMService rmService = rm.getClientRMService(); testApplicationPriorityUpdation(rmService, app1, appPriority, appPriority); rm.killApp(app1.getApplicationId()); @@ -2278,7 +2285,10 @@ public void testUpdateApplicationPriorityRequest() throws Exception { rm.start(); rm.registerNode("host1:1234", 1024); // Start app1 with appPriority 5 - RMApp app1 = rm.submitApp(1024, Priority.newInstance(appPriority)); + MockRMAppSubmissionData data = MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm) + .withPriority(Priority.newInstance(appPriority)) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); Assert.assertEquals("Incorrect priority has been set to application", appPriority, app1.getApplicationPriority().getPriority()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java index a6b30b3f6b3..941f5c1a0e5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java @@ -80,7 +80,7 @@ public void testUsageWithOneAttemptAndOneContainer() throws Exception { new MockNM("127.0.0.1:1234", 15120, rm.getResourceTrackerService()); nm.registerNode(); - RMApp app0 = rm.submitApp(200); + RMApp app0 = MockRMAppSubmitter.submitWithMemory(200, rm); RMAppMetrics rmAppMetrics = app0.getRMAppMetrics(); Assert.assertTrue( @@ -149,7 +149,7 @@ public void testUsageWithMultipleContainersAndRMRestart() throws Exception { new MockNM("127.0.0.1:1234", 65536, rm0.getResourceTrackerService()); nm.registerNode(); - RMApp app0 = rm0.submitApp(200); + RMApp app0 = MockRMAppSubmitter.submitWithMemory(200, rm0); rm0.waitForState(app0.getApplicationId(), RMAppState.ACCEPTED); RMAppAttempt attempt0 = app0.getCurrentAppAttempt(); @@ -264,10 +264,21 @@ private void amRestartTests(boolean keepRunningContainers) MockRM rm = new MockRM(conf); rm.start(); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm) + .withName("name") + .withUser("user") + .withAcls(new HashMap()) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(-1) + .withCredentials(null) + .withAppType("MAPREDUCE") + .withWaitForAccepted(false) + .withKeepContainers(keepRunningContainers) + .build(); RMApp app = - rm.submitApp(200, "name", "user", - new HashMap(), false, "default", -1, - null, "MAPREDUCE", false, keepRunningContainers); + MockRMAppSubmitter.submit(rm, data); MockNM nm = new MockNM("127.0.0.1:1234", 10240, rm.getResourceTrackerService()); nm.registerNode(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestDecommissioningNodesWatcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestDecommissioningNodesWatcher.java index 43711560855..9ae44ca579b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestDecommissioningNodesWatcher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestDecommissioningNodesWatcher.java @@ -64,7 +64,7 @@ public void testDecommissioningNodesWatcher() throws Exception { rm.waitForState(id1, NodeState.RUNNING); Assert.assertFalse(watcher.checkReadyToBeDecommissioned(id1)); - RMApp app = rm.submitApp(2000); + RMApp app = MockRMAppSubmitter.submitWithMemory(2000, rm); MockAM am = MockRM.launchAndRegisterAM(app, rm, nm1); // Setup nm1 as DECOMMISSIONING for DecommissioningNodesWatcher. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestKillApplicationWithRMHA.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestKillApplicationWithRMHA.java index 149ddf57371..6d78e0cc2d5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestKillApplicationWithRMHA.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestKillApplicationWithRMHA.java @@ -21,6 +21,7 @@ import static org.junit.Assert.fail; import java.io.IOException; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -61,12 +62,23 @@ public void testKillAppWhenFailoverHappensAtNewState() nm1.registerNode(); // Submit the application + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("") + .withUser(UserGroupInformation + .getCurrentUser().getShortUserName()) + .withAcls(null) + .withUnmanaged(false) + .withQueue(null) + .withMaxAppAttempts(configuration.getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, + YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS)) + .withCredentials(null) + .withAppType(null) + .withWaitForAccepted(false) + .withKeepContainers(false) + .build(); RMApp app0 = - rm1.submitApp(200, "", UserGroupInformation - .getCurrentUser().getShortUserName(), null, false, null, - configuration.getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, - false, false); + MockRMAppSubmitter.submit(rm1, data); // failover and kill application // When FailOver happens, the state of this application is NEW, @@ -92,7 +104,7 @@ public void testKillAppWhenFailoverHappensAtRunningState() nm1.registerNode(); // create app and launch the AM - RMApp app0 = rm1.submitApp(200); + RMApp app0 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am0 = launchAM(app0, rm1, nm1); // failover and kill application @@ -115,7 +127,7 @@ public void testKillAppWhenFailoverHappensAtFinalState() nm1.registerNode(); // create app and launch the AM - RMApp app0 = rm1.submitApp(200); + RMApp app0 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am0 = launchAM(app0, rm1, nm1); // kill the app. @@ -146,7 +158,7 @@ public void testKillAppWhenFailOverHappensDuringApplicationKill() nm1.registerNode(); // create app and launch the AM - RMApp app0 = rm1.submitApp(200); + RMApp app0 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am0 = launchAM(app0, rm1, nm1); // ensure that the app is in running state diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestLeaderElectorService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestLeaderElectorService.java index 121cacba5e7..ab07139cf91 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestLeaderElectorService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestLeaderElectorService.java @@ -129,7 +129,14 @@ public synchronized void storeApplicationStateInternal(ApplicationId rm2 = startRM("rm2", HAServiceState.STANDBY); // submit an app which will trigger state-store failure. - rm1.submitApp(200, "app1", "user1", null, "default", false); + MockRMAppSubmitter.submit(rm1, + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("app1") + .withUser("user1") + .withAcls(null) + .withQueue("default") + .withWaitForAccepted(false) + .build()); waitFor(rm1, HAServiceState.STANDBY); // rm2 should become active; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestMoveApplication.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestMoveApplication.java index 05b25df3a33..a078fb2ed7b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestMoveApplication.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestMoveApplication.java @@ -124,7 +124,7 @@ public void testMoveTooLate() throws Exception { void testMoveSuccessful() throws Exception { MockRM rm1 = new MockRM(conf); rm1.start(); - RMApp app = rm1.submitApp(1024); + RMApp app = MockRMAppSubmitter.submitWithMemory(1024, rm1); ClientRMService clientRMService = rm1.getClientRMService(); // FIFO scheduler does not support moves clientRMService diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestNodeBlacklistingOnAMFailures.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestNodeBlacklistingOnAMFailures.java index e7d666ad8ad..1ac11baf98a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestNodeBlacklistingOnAMFailures.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestNodeBlacklistingOnAMFailures.java @@ -58,6 +58,33 @@ */ public class TestNodeBlacklistingOnAMFailures { + private RMApp submitAppWithAMResourceRequests(MockRM rm, List amResourceRequests) throws Exception { + int maxAttempts = rm.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS); + MockRMAppSubmissionData data = MockRMAppSubmissionData.Builder.create() + .withAmResourceRequests(amResourceRequests) + .withName("app1") + .withUser("user") + .withAcls(null) + .withUnmanaged(false) + .withQueue(null) + .withMaxAppAttempts(maxAttempts) + .withCredentials(null) + .withAppType(null) + .withWaitForAccepted(true) + .withKeepContainers(false) + .withIsAppIdProvided(false) + .withApplicationId(null) + .withAttemptFailuresValidityInterval(0) + .withLogAggregationContext(null) + .withCancelTokensWhenComplete(true) + .withPriority(amResourceRequests.get(0).getPriority()) + .withAmLabel(amResourceRequests.get(0).getNodeLabelExpression()) + .withApplicationTimeouts(null) + .withTokensConf(null) + .build(); + return MockRMAppSubmitter.submit(rm, data); + } + @Before public void setup() { QueueMetrics.clearQueueMetrics(); @@ -87,7 +114,7 @@ public void testNodeBlacklistingOnAMFailure() throws Exception { new MockNM("127.0.0.2:2345", 8000, rm.getResourceTrackerService()); nm2.registerNode(); - RMApp app = rm.submitApp(200); + RMApp app = MockRMAppSubmitter.submitWithMemory(200, rm); MockAM am1 = MockRM.launchAndRegisterAM(app, rm, nm1); ContainerId amContainerId = @@ -215,7 +242,7 @@ public void testNodeBlacklistingOnAMFailureStrictNodeLocality() reqs.add(anyReq); reqs.add(rackReq); reqs.add(nodeReq); - RMApp app = rm.submitApp(reqs); + RMApp app = submitAppWithAMResourceRequests(rm, reqs); MockAM am1 = MockRM.launchAndRegisterAM(app, rm, nm2); ContainerId amContainerId = @@ -302,7 +329,7 @@ public void testNodeBlacklistingOnAMFailureRelaxedNodeLocality() reqs.add(anyReq); reqs.add(rackReq); reqs.add(nodeReq); - RMApp app = rm.submitApp(reqs); + RMApp app = submitAppWithAMResourceRequests(rm, reqs); MockAM am1 = MockRM.launchAndRegisterAM(app, rm, nm2); ContainerId amContainerId = @@ -363,7 +390,7 @@ public void testNoBlacklistingForNonSystemErrors() throws Exception { new MockNM("127.0.0.1:1234", 8000, rm.getResourceTrackerService()); node.registerNode(); - RMApp app = rm.submitApp(200); + RMApp app = MockRMAppSubmitter.submitWithMemory(200, rm); ApplicationId appId = app.getApplicationId(); int numAppAttempts = 1; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestOpportunisticContainerAllocatorAMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestOpportunisticContainerAllocatorAMService.java index 5542157c940..f6f11d0467a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestOpportunisticContainerAllocatorAMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestOpportunisticContainerAllocatorAMService.java @@ -177,7 +177,15 @@ public void testContainerPromoteAndDemoteBeforeContainerStart() throws Exception OpportunisticContainerAllocatorAMService amservice = (OpportunisticContainerAllocatorAMService) rm .getApplicationMasterService(); - RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); ApplicationAttemptId attemptId = app1.getCurrentAppAttempt().getAppAttemptId(); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm2); @@ -344,7 +352,15 @@ public void testContainerPromoteAfterContainerStart() throws Exception { OpportunisticContainerAllocatorAMService amservice = (OpportunisticContainerAllocatorAMService) rm .getApplicationMasterService(); - RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); ApplicationAttemptId attemptId = app1.getCurrentAppAttempt().getAppAttemptId(); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm2); @@ -469,7 +485,15 @@ public void testContainerPromoteAfterContainerComplete() throws Exception { OpportunisticContainerAllocatorAMService amservice = (OpportunisticContainerAllocatorAMService) rm .getApplicationMasterService(); - RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); ApplicationAttemptId attemptId = app1.getCurrentAppAttempt().getAppAttemptId(); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm2); @@ -582,7 +606,15 @@ public void testContainerAutoUpdateContainer() throws Exception { OpportunisticContainerAllocatorAMService amservice = (OpportunisticContainerAllocatorAMService) rm .getApplicationMasterService(); - RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); ApplicationAttemptId attemptId = app1.getCurrentAppAttempt().getAppAttemptId(); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); @@ -746,7 +778,15 @@ public void testNodeRemovalDuringAllocate() throws Exception { OpportunisticContainerAllocatorAMService amservice = (OpportunisticContainerAllocatorAMService) rm .getApplicationMasterService(); - RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); ApplicationAttemptId attemptId = app1.getCurrentAppAttempt().getAppAttemptId(); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm2); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java index 36793190315..0e17bc1b158 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java @@ -126,8 +126,8 @@ public void testAppWithNoContainers() throws Exception { MockRM rm = new MockRM(conf); rm.start(); MockNM nm1 = rm.registerNode("h1:1234", 5120); - - RMApp app = rm.submitApp(2000); + + RMApp app = MockRMAppSubmitter.submitWithMemory(2000, rm); //kick the scheduling nm1.nodeHeartbeat(true); @@ -150,8 +150,8 @@ public void testAppOnMultiNode() throws Exception { rm.start(); MockNM nm1 = rm.registerNode("h1:1234", 5120); MockNM nm2 = rm.registerNode("h2:5678", 10240); - - RMApp app = rm.submitApp(2000); + + RMApp app = MockRMAppSubmitter.submitWithMemory(2000, rm); //kick the scheduling nm1.nodeHeartbeat(true); @@ -207,7 +207,7 @@ public void testNMTokenSentForNormalContainer() throws Exception { MockRM rm = new MockRM(conf); rm.start(); MockNM nm1 = rm.registerNode("h1:1234", 5120); - RMApp app = rm.submitApp(2000); + RMApp app = MockRMAppSubmitter.submitWithMemory(2000, rm); RMAppAttempt attempt = app.getCurrentAppAttempt(); // Call getNewContainerId to increase container Id so that the AM container @@ -263,7 +263,7 @@ public void testNMToken() throws Exception { rm.getRMContext().getNMTokenSecretManager(); // submitting new application - RMApp app = rm.submitApp(1000); + RMApp app = MockRMAppSubmitter.submitWithMemory(1000, rm); // start scheduling. nm1.nodeHeartbeat(true); @@ -444,10 +444,10 @@ public void testActivatingApplicationAfterAddingNM() throws Exception { rm1.start(); // app that gets launched - RMApp app1 = rm1.submitApp(200); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(200, rm1); // app that does not get launched - RMApp app2 = rm1.submitApp(200); + RMApp app2 = MockRMAppSubmitter.submitWithMemory(200, rm1); // app1 and app2 should be scheduled, but because no resource is available, // they are not activated. @@ -488,7 +488,7 @@ public void testInvalidateAMHostPortWhenAMFailedOrKilled() throws Exception { rm1.start(); // a succeeded app - RMApp app1 = rm1.submitApp(200); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService()); nm1.registerNode(); @@ -496,14 +496,14 @@ public void testInvalidateAMHostPortWhenAMFailedOrKilled() throws Exception { MockRM.finishAMAndVerifyAppState(app1, rm1, nm1, am1); // a failed app - RMApp app2 = rm1.submitApp(200); + RMApp app2 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1); nm1.nodeHeartbeat(am2.getApplicationAttemptId(), 1, ContainerState.COMPLETE); rm1.waitForState(am2.getApplicationAttemptId(), RMAppAttemptState.FAILED); rm1.waitForState(app2.getApplicationId(), RMAppState.FAILED); // a killed app - RMApp app3 = rm1.submitApp(200); + RMApp app3 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am3 = MockRM.launchAndRegisterAM(app3, rm1, nm1); rm1.killApp(app3.getApplicationId()); rm1.waitForState(app3.getApplicationId(), RMAppState.KILLED); @@ -542,7 +542,7 @@ public void testInvalidatedAMHostPortOnAMRestart() throws Exception { nm1.registerNode(); // a failed app - RMApp app2 = rm1.submitApp(200); + RMApp app2 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1); nm1 .nodeHeartbeat(am2.getApplicationAttemptId(), 1, ContainerState.COMPLETE); @@ -609,7 +609,7 @@ protected Dispatcher createDispatcher() { nm1.registerNode(); // a failed app - RMApp application = rm.submitApp(200); + RMApp application = MockRMAppSubmitter.submitWithMemory(200, rm); MockAM am = MockRM.launchAM(application, rm, nm1); rm.waitForState(am.getApplicationAttemptId(), RMAppAttemptState.LAUNCHED); nm1.nodeHeartbeat(am.getApplicationAttemptId(), 1, ContainerState.RUNNING); @@ -687,7 +687,7 @@ protected Dispatcher createDispatcher() { MockNM nm1 = new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService()); nm1.registerNode(); - RMApp app1 = rm1.submitApp(200); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); rm1.killApp(app1.getApplicationId()); @@ -741,7 +741,7 @@ protected Dispatcher createDispatcher() { MockNM nm1 = new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService()); nm1.registerNode(); - RMApp app1 = rm1.submitApp(200); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); rm1.killApp(app1.getApplicationId()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java index c17dee80d37..2ccd07f2bd0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java @@ -141,7 +141,7 @@ private void checkActiveRMFunctionality() throws Exception { try { rm.getNewAppId(); nm = rm.registerNode("127.0.0.1:1", 2048); - app = rm.submitApp(1024); + app = MockRMAppSubmitter.submitWithMemory(1024, rm); attempt = app.getCurrentAppAttempt(); rm.waitForState(attempt.getAppAttemptId(), RMAppAttemptState.SCHEDULED); } catch (Exception e) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHAForAsyncScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHAForAsyncScheduler.java index 36f17623281..d46b8266c2e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHAForAsyncScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHAForAsyncScheduler.java @@ -136,11 +136,21 @@ public void testAsyncScheduleThreadStateAfterRMHATransit() throws Exception { } private RMApp submitAppAndCheckLaunched(MockRM rm) throws Exception { - RMApp app = rm.submitApp(200, "", - UserGroupInformation.getCurrentUser().getShortUserName(), null, false, - "default", configuration.getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, false, - false); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm) + .withName("") + .withUser(UserGroupInformation.getCurrentUser().getShortUserName()) + .withAcls(null) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(configuration.getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, + YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS)) + .withCredentials(null) + .withAppType(null) + .withWaitForAccepted(false) + .withKeepContainers(false) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); rm.waitForState(app.getApplicationId(), RMAppState.ACCEPTED); RMAppAttempt attempt = app.getCurrentAppAttempt(); rm.sendAMLaunched(attempt.getAppAttemptId()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHATimelineCollectors.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHATimelineCollectors.java index a0cd0486d3a..d4f156bd0b1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHATimelineCollectors.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHATimelineCollectors.java @@ -60,14 +60,14 @@ public void testRebuildCollectorDataOnFailover() throws Exception { = new MockNM("127.0.0.1:1234", 15120, rm2.getResourceTrackerService()); MockNM nm2 = new MockNM("127.0.0.1:5678", 15121, rm2.getResourceTrackerService()); - RMApp app1 = rm1.submitApp(1024); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(1024, rm1); String collectorAddr1 = "1.2.3.4:5"; AppCollectorData data1 = AppCollectorData.newInstance( app1.getApplicationId(), collectorAddr1); nm1.addRegisteringCollector(app1.getApplicationId(), data1); String collectorAddr2 = "5.4.3.2:1"; - RMApp app2 = rm1.submitApp(1024); + RMApp app2 = MockRMAppSubmitter.submitWithMemory(1024, rm1); AppCollectorData data2 = AppCollectorData.newInstance( app2.getApplicationId(), collectorAddr2, rm1.getStartTime(), 1); nm1.addRegisteringCollector(app2.getApplicationId(), data2); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java index d5880ca9e4e..423c88d49fe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java @@ -194,6 +194,36 @@ private MockRM createMockRM(YarnConfiguration config) { return rm; } + private RMApp submitApp(MockRM rm, List amResourceRequests, + String appNodeLabel) throws Exception { + int maxAttempts = rm.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS); + MockRMAppSubmissionData data = MockRMAppSubmissionData.Builder.create() + .withAmResourceRequests(amResourceRequests) + .withName("app1") + .withUser("user") + .withAcls(null) + .withUnmanaged(false) + .withQueue(null) + .withMaxAppAttempts(maxAttempts) + .withCredentials(null) + .withAppType(null) + .withWaitForAccepted(true) + .withKeepContainers(false) + .withIsAppIdProvided(false) + .withApplicationId(null) + .withAttemptFailuresValidityInterval(0) + .withLogAggregationContext(null) + .withCancelTokensWhenComplete(true) + .withPriority(amResourceRequests.get(0).getPriority()) + .withAmLabel(amResourceRequests.get(0).getNodeLabelExpression()) + .withApplicationTimeouts(null) + .withTokensConf(null) + .withApplicationTags(null) + .withAppNodeLabel(appNodeLabel) + .build(); + return MockRMAppSubmitter.submit(rm, data); + } + @Test (timeout=180000) public void testRMRestart() throws Exception { conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, @@ -217,7 +247,7 @@ public void testRMRestart() throws Exception { nm2.registerNode(); // nm2 will not heartbeat with RM1 // create app that will finish and the final state should be saved. - RMApp app0 = rm1.submitApp(200); + RMApp app0 = MockRMAppSubmitter.submitWithMemory(200, rm1); RMAppAttempt attempt0 = app0.getCurrentAppAttempt(); // spot check that app is saved Assert.assertEquals(1, rmAppState.size()); @@ -227,7 +257,7 @@ public void testRMRestart() throws Exception { finishApplicationMaster(app0, rm1, nm1, am0); // create app that gets launched and does allocate before RM restart - RMApp app1 = rm1.submitApp(200); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(200, rm1); // assert app1 info is saved ApplicationStateData appState = rmAppState.get(app1.getApplicationId()); Assert.assertNotNull(appState); @@ -268,7 +298,7 @@ public void testRMRestart() throws Exception { } // create app that does not get launched by RM before RM restart - RMApp app2 = rm1.submitApp(200); + RMApp app2 = MockRMAppSubmitter.submitWithMemory(200, rm1); // assert app2 info is saved appState = rmAppState.get(app2.getApplicationId()); @@ -279,9 +309,17 @@ public void testRMRestart() throws Exception { .getApplicationId()); // create unmanaged app - RMApp appUnmanaged = rm1.submitApp(200, "someApp", "someUser", null, true, - null, conf.getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null); + RMApp appUnmanaged = MockRMAppSubmitter.submit(rm1, + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("someApp") + .withUser("someUser") + .withAcls(null) + .withUnmanaged(true) + .withQueue(null) + .withMaxAppAttempts(conf.getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, + YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS)) + .withCredentials(null) + .build()); ApplicationAttemptId unmanagedAttemptId = appUnmanaged.getCurrentAppAttempt().getAppAttemptId(); // assert appUnmanaged info is saved @@ -483,7 +521,7 @@ public void testAppReportNodeLabelRMRestart() throws Exception { Resource.newInstance(200, 1), 1, true, amLabel.getName()); ArrayList resReqs = new ArrayList<>(); resReqs.add(amResourceRequest); - RMApp app0 = rm1.submitApp(resReqs, appLabel.getName()); + RMApp app0 = submitApp(rm1, resReqs, appLabel.getName()); rm1.killApp(app0.getApplicationId()); rm1.waitForState(app0.getApplicationId(), RMAppState.KILLED); // start new RM @@ -509,7 +547,14 @@ public void testUnManagedRMRestart() throws Exception { rm1.start(); // create app and launch the AM RMApp app0 = - rm1.submitApp(null, "name", "user", new HashMap<>(), true, "default"); + MockRMAppSubmitter.submit(rm1, + MockRMAppSubmissionData.Builder.createWithDefaults(null, rm1) + .withName("name") + .withUser("user") + .withAcls(new HashMap<>()) + .withUnmanaged(true) + .withQueue("default") + .build()); rm1.killApp(app0.getApplicationId()); rm1.waitForState(app0.getApplicationId(), RMAppState.KILLED); // start new RM @@ -540,10 +585,21 @@ public void testRMRestartAppRunningAMFailed() throws Exception { nm1.registerNode(); // create app and launch the AM + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("name") + .withUser("user") + .withAcls(new HashMap()) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(-1) + .withCredentials(null) + .withAppType("MAPREDUCE") + .withWaitForAccepted(true) + .withKeepContainers(true) + .build(); RMApp app0 = - rm1.submitApp(200, "name", "user", - new HashMap(), false, "default", -1, - null, "MAPREDUCE", true, true); + MockRMAppSubmitter.submit(rm1, data); MockAM am0 = launchAM(app0, rm1, nm1); // fail the AM by sending CONTAINER_FINISHED event without registering. @@ -598,7 +654,7 @@ public void testRMRestartWaitForPreviousAMToFinish() throws Exception { nm1.registerNode(); // submitting app - RMApp app1 = rm1.submitApp(200); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(200, rm1); rm1.waitForState(app1.getApplicationId(), RMAppState.ACCEPTED); MockAM am1 = launchAM(app1, rm1, nm1); nm1.nodeHeartbeat(am1.getApplicationAttemptId(), 1, ContainerState.COMPLETE); @@ -706,7 +762,7 @@ public Boolean get() { // It will be saved only when we launch AM. // submitting app but not starting AM for it. - RMApp app2 = rm3.submitApp(200); + RMApp app2 = MockRMAppSubmitter.submitWithMemory(200, rm3); rm3.waitForState(app2.getApplicationId(), RMAppState.ACCEPTED); Assert.assertEquals(1, app2.getAppAttempts().size()); Assert.assertEquals(0, @@ -772,7 +828,7 @@ public void updateApplicationStateInternal(ApplicationId appId, MockRM rm1 = createMockRM(conf, memStore); rm1.start(); MockNM nm1 = rm1.registerNode("127.0.0.1:1234", 15120); - RMApp app0 = rm1.submitApp(200); + RMApp app0 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am0 = MockRM.launchAndRegisterAM(app0, rm1, nm1); FinishApplicationMasterRequest req = @@ -814,7 +870,7 @@ public void testRMRestartFailedApp() throws Exception { nm1.registerNode(); // create app and launch the AM - RMApp app0 = rm1.submitApp(200); + RMApp app0 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am0 = launchAM(app0, rm1, nm1); // fail the AM by sending CONTAINER_FINISHED event without registering. @@ -862,7 +918,7 @@ public void testRMRestartKilledApp() throws Exception{ nm1.registerNode(); // create app and launch the AM - RMApp app0 = rm1.submitApp(200); + RMApp app0 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am0 = launchAM(app0, rm1, nm1); // kill the app. @@ -918,9 +974,18 @@ public synchronized void updateApplicationAttemptStateInternal( rm1.start(); // create app RMApp app0 = - rm1.submitApp(200, "name", "user", - new HashMap(), false, "default", -1, - null, "MAPREDUCE", false); + MockRMAppSubmitter.submit(rm1, + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("name") + .withUser("user") + .withAcls(new HashMap()) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(-1) + .withCredentials(null) + .withAppType("MAPREDUCE") + .withWaitForAccepted(false) + .build()); // kill the app. rm1.killApp(app0.getApplicationId()); rm1.waitForState(app0.getApplicationId(), RMAppState.KILLED); @@ -952,7 +1017,7 @@ public void testRMRestartSucceededApp() throws Exception { nm1.registerNode(); // create an app and finish the app. - RMApp app0 = rm1.submitApp(200); + RMApp app0 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am0 = launchAM(app0, rm1, nm1); // unregister am @@ -1003,14 +1068,32 @@ protected SystemMetricsPublisher createSystemMetricsPublisher() { (MockMemoryRMStateStore) rm1.getRMStateStore(); // a succeeded app. - RMApp app0 = rm1.submitApp(200, "name", "user", null, - false, "default", 1, null, "myType"); + RMApp app0 = MockRMAppSubmitter.submit(rm1, + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("name") + .withUser("user") + .withAcls(null) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(1) + .withCredentials(null) + .withAppType("myType") + .build()); MockAM am0 = launchAM(app0, rm1, nm1); finishApplicationMaster(app0, rm1, nm1, am0); // a failed app. - RMApp app1 = rm1.submitApp(200, "name", "user", null, - false, "default", 1, null, "myType"); + RMApp app1 = MockRMAppSubmitter.submit(rm1, + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("name") + .withUser("user") + .withAcls(null) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(1) + .withCredentials(null) + .withAppType("myType") + .build()); MockAM am1 = launchAM(app1, rm1, nm1); // fail the AM by sending CONTAINER_FINISHED event without registering. nm1.nodeHeartbeat(am1.getApplicationAttemptId(), 1, ContainerState.COMPLETE); @@ -1018,8 +1101,17 @@ protected SystemMetricsPublisher createSystemMetricsPublisher() { rm1.waitForState(app1.getApplicationId(), RMAppState.FAILED); // a killed app. - RMApp app2 = rm1.submitApp(200, "name", "user", null, - false, "default", 1, null, "myType"); + RMApp app2 = MockRMAppSubmitter.submit(rm1, + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("name") + .withUser("user") + .withAcls(null) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(1) + .withCredentials(null) + .withAppType("myType") + .build()); MockAM am2 = launchAM(app2, rm1, nm1); rm1.killApp(app2.getApplicationId()); rm1.waitForState(app2.getApplicationId(), RMAppState.KILLED); @@ -1162,13 +1254,27 @@ public void testRMRestartOnMaxAppAttempts() throws Exception { nm1.registerNode(); // submit an app with maxAppAttempts equals to 1 - RMApp app1 = rm1.submitApp(200, "name", "user", - new HashMap(), false, "default", 1, - null); + RMApp app1 = MockRMAppSubmitter.submit(rm1, + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("name") + .withUser("user") + .withAcls(new HashMap()) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(1) + .withCredentials(null) + .build()); // submit an app with maxAppAttempts equals to -1 - RMApp app2 = rm1.submitApp(200, "name", "user", - new HashMap(), false, "default", -1, - null); + RMApp app2 = MockRMAppSubmitter.submit(rm1, + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("name") + .withUser("user") + .withAcls(new HashMap()) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(-1) + .withCredentials(null) + .build()); // assert app1 info is saved ApplicationStateData appState = rmAppState.get(app1.getApplicationId()); @@ -1232,9 +1338,16 @@ public void testRMRestartTimelineCollectorContext() throws Exception { nm1.registerNode(); // submit an app. - RMApp app = rm1.submitApp(200, "name", "user", - new HashMap(), false, "default", -1, - null); + RMApp app = MockRMAppSubmitter.submit(rm1, + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("name") + .withUser("user") + .withAcls(new HashMap()) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(-1) + .withCredentials(null) + .build()); // Check if app info has been saved. ApplicationStateData appState = rmAppState.get(app.getApplicationId()); Assert.assertNotNull(appState); @@ -1321,8 +1434,16 @@ public void testDelegationTokenRestoredInDelegationTokenRenewer() tokenSet.add(token2); // submit an app with customized credential - RMApp app = rm1.submitApp(200, "name", "user", - new HashMap(), false, "default", 1, ts); + RMApp app = MockRMAppSubmitter.submit(rm1, + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("name") + .withUser("user") + .withAcls(new HashMap()) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(1) + .withCredentials(ts) + .build()); // assert app info is saved ApplicationStateData appState = rmAppState.get(app.getApplicationId()); @@ -1389,9 +1510,16 @@ public void testAppAttemptTokensRestoredOnRMRestart() throws Exception { nm1.registerNode(); // submit an app + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("name") + .withUser("user") + .withAcls(new HashMap()) + .withQueue("default") + .withUnmanaged(false) + .build(); RMApp app1 = - rm1.submitApp(200, "name", "user", - new HashMap(), "default"); + MockRMAppSubmitter.submit(rm1, data); // assert app info is saved ApplicationStateData appState = rmAppState.get(app1.getApplicationId()); @@ -1492,8 +1620,16 @@ public void testRMDelegationTokenRestoredOnRMRestart() throws Exception { tokenIdentSet.add(dtId1); // submit an app with customized credential - RMApp app = rm1.submitApp(200, "name", "user", - new HashMap(), false, "default", 1, ts); + RMApp app = MockRMAppSubmitter.submit(rm1, + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("name") + .withUser("user") + .withAcls(new HashMap()) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(1) + .withCredentials(ts) + .build()); // assert app info is saved ApplicationStateData appState = rmAppState.get(app.getApplicationId()); @@ -1624,8 +1760,16 @@ public void testAppSubmissionWithOldDelegationTokenAfterRMRestart() // submit an app with the old delegation token got from previous RM. Credentials ts = new Credentials(); ts.addToken(token1.getService(), token1); - RMApp app = rm2.submitApp(200, "name", "user", - new HashMap(), false, "default", 1, ts); + RMApp app = MockRMAppSubmitter.submit(rm2, + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm2) + .withName("name") + .withUser("user") + .withAcls(new HashMap()) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(1) + .withCredentials(ts) + .build()); rm2.waitForState(app.getApplicationId(), RMAppState.ACCEPTED); } @@ -1664,9 +1808,18 @@ protected void handleStoreEvent(RMStateStoreEvent event) { final int NUM_APPS = 5; for (int i = 0; i < NUM_APPS; i++) { - RMApp app = rm1.submitApp(200, "name", "user", - new HashMap(), false, - "default", -1, null, "MAPREDUCE", false); + RMApp app = MockRMAppSubmitter.submit(rm1, + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("name") + .withUser("user") + .withAcls(new HashMap()) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(-1) + .withCredentials(null) + .withAppType("MAPREDUCE") + .withWaitForAccepted(false) + .build()); appList.add(app); rm1.waitForState(app.getApplicationId(), RMAppState.NEW_SAVING); } @@ -1709,7 +1862,7 @@ public void testFinishedAppRemovalAfterRMRestart() throws Exception { nm1.registerNode(); // create an app and finish the app. - RMApp app0 = rm1.submitApp(200); + RMApp app0 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am0 = launchAM(app0, rm1, nm1); finishApplicationMaster(app0, rm1, nm1, am0); @@ -1727,7 +1880,7 @@ public void testFinishedAppRemovalAfterRMRestart() throws Exception { rm2.waitForState(app0.getApplicationId(), RMAppState.FINISHED); // create one more app and finish the app. - RMApp app1 = rm2.submitApp(200); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(200, rm2); MockAM am1 = launchAM(app1, rm2, nm1); finishApplicationMaster(app1, rm2, nm1, am1); rm2.drainEvents(); @@ -1777,8 +1930,17 @@ public void testClientRetryOnKillingApplication() throws Exception { nm1.registerNode(); RMApp app1 = - rm1.submitApp(200, "name", "user", null, false, "default", 1, null, - "myType"); + MockRMAppSubmitter.submit(rm1, + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("name") + .withUser("user") + .withAcls(null) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(1) + .withCredentials(null) + .withAppType("myType") + .build()); MockAM am1 = launchAM(app1, rm1, nm1); KillApplicationResponse response; @@ -1824,9 +1986,18 @@ protected DelegationTokenRenewer createDelegationTokenRenewer() { (MockMemoryRMStateStore) rm1.getRMStateStore(); RMApp app1 = null; try { - app1 = rm1.submitApp(200, "name", "user", - new HashMap(), false, "default", -1, - null, "MAPREDUCE", false); + app1 = MockRMAppSubmitter.submit(rm1, + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("name") + .withUser("user") + .withAcls(new HashMap()) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(-1) + .withCredentials(null) + .withAppType("MAPREDUCE") + .withWaitForAccepted(false) + .build()); Assert.fail(); } catch (Exception e) { @@ -1911,7 +2082,7 @@ public void testQueueMetricsOnRMRestart() throws Exception { assertQueueMetrics(qm1, 0, 0, 0, 0); // create app that gets launched and does allocate before RM restart - RMApp app1 = rm1.submitApp(200); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(200, rm1); // Need to wait first for AppAttempt to be started (RMAppState.ACCEPTED) // and then for it to reach RMAppAttemptState.SCHEDULED // inorder to ensure appsPending metric is incremented @@ -2105,7 +2276,7 @@ public void testSynchronouslyRenewDTOnRecovery() throws Exception { final MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService()); nm1.registerNode(); - RMApp app0 = rm1.submitApp(200); + RMApp app0 = MockRMAppSubmitter.submitWithMemory(200, rm1); final MockAM am0 = MockRM.launchAndRegisterAM(app0, rm1, nm1); MockRM rm2 = new MockRM(conf, rm1.getRMStateStore()) { @@ -2362,7 +2533,7 @@ public void testRMRestartFailAppAttempt() throws Exception { nm1.registerNode(); // create app and launch the AM - RMApp app0 = rm1.submitApp(200); + RMApp app0 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am0 = launchAM(app0, rm1, nm1); ApplicationId applicationId = app0.getApplicationId(); @@ -2491,7 +2662,7 @@ public void testRMRestartAfterPreemption() throws Exception { nm1.registerNode(); int CONTAINER_MEMORY = 1024; // create app and launch the AM - RMApp app0 = rm1.submitApp(CONTAINER_MEMORY); + RMApp app0 = MockRMAppSubmitter.submitWithMemory(CONTAINER_MEMORY, rm1); MockAM am0 = MockRM.launchAM(app0, rm1, nm1); nm1.nodeHeartbeat(am0.getApplicationAttemptId(), 1, ContainerState.COMPLETE); @@ -2545,7 +2716,7 @@ public void testRMRestartOnMissingAttempts() throws Exception { nm1.registerNode(); // create an app and finish the app. - RMApp app0 = rm1.submitApp(200); + RMApp app0 = MockRMAppSubmitter.submitWithMemory(200, rm1); ApplicationStateData app0State = memStore.getState().getApplicationState() .get(app0.getApplicationId()); @@ -2637,7 +2808,15 @@ protected RMNodeLabelsManager createNodeLabelManager() { MockNM nm1 = rm1.registerNode("h1:1234", 8000); // label = x // submit an application with specifying am node label expression as "x" - RMApp app1 = rm1.submitApp(200, "someApp", "someUser", null, "a1", "x"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("someApp") + .withUser("someUser") + .withAcls(null) + .withQueue("a1") + .withAmLabel("x") + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); // check am container allocated with correct node label expression MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); ContainerId amContainerId1 = @@ -2703,8 +2882,13 @@ public void testRMRestartAfterPriorityChangesInAllocatedResponse() // Submit an application Priority appPriority1 = Priority.newInstance(5); - RMApp app1 = rm.submitApp(2048, appPriority1, - getCreds(), getTokensConf()); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(2048, rm) + .withPriority(appPriority1) + .withCredentials(getCreds()) + .withTokensConf(getTokensConf()) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); nm1.nodeHeartbeat(true); RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); @@ -2776,7 +2960,7 @@ public void testRMRestartAfterUpdateTrackingUrl() throws Exception { // Register node1 MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * 1024); - RMApp app1 = rm.submitApp(2048); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(2048, rm); nm1.nodeHeartbeat(true); RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java index b451db1a7d5..0b937e4f224 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java @@ -396,7 +396,7 @@ public void testGracefulDecommissionWithApp() throws Exception { rm.waitForState(id3, NodeState.RUNNING); // Create an app and launch two containers on host1. - RMApp app = rm.submitApp(2000); + RMApp app = MockRMAppSubmitter.submitWithMemory(2000, rm); MockAM am = MockRM.launchAndRegisterAM(app, rm, nm1); ApplicationAttemptId aaid = app.getCurrentAppAttempt().getAppAttemptId(); nm1.nodeHeartbeat(aaid, 2, ContainerState.RUNNING); @@ -1187,13 +1187,13 @@ public void testNodeHeartbeatForAppCollectorsMap() throws Exception { RMNodeImpl node2 = (RMNodeImpl) rm.getRMContext().getRMNodes().get(nm2.getNodeId()); - RMAppImpl app1 = (RMAppImpl) rm.submitApp(1024); + RMAppImpl app1 = (RMAppImpl) MockRMAppSubmitter.submitWithMemory(1024, rm); String collectorAddr1 = "1.2.3.4:5"; app1.setCollectorData(AppCollectorData.newInstance( app1.getApplicationId(), collectorAddr1)); String collectorAddr2 = "5.4.3.2:1"; - RMAppImpl app2 = (RMAppImpl) rm.submitApp(1024); + RMAppImpl app2 = (RMAppImpl) MockRMAppSubmitter.submitWithMemory(1024, rm); app2.setCollectorData(AppCollectorData.newInstance( app2.getApplicationId(), collectorAddr2)); @@ -1317,7 +1317,11 @@ public void testHandleContainerStatusInvalidCompletions() throws Exception { spy(rm.getRMContext().getDispatcher().getEventHandler()); // Case 1: Unmanaged AM - RMApp app = rm.submitApp(1024, true); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm) + .withUnmanaged(true) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); // Case 1.1: AppAttemptId is null NMContainerStatus report = @@ -1341,7 +1345,7 @@ public void testHandleContainerStatusInvalidCompletions() throws Exception { verify(handler, never()).handle((Event)any()); // Case 2: Managed AM - app = rm.submitApp(1024); + app = MockRMAppSubmitter.submitWithMemory(1024, rm); // Case 2.1: AppAttemptId is null report = NMContainerStatus.newInstance( @@ -2245,7 +2249,11 @@ protected Dispatcher createDispatcher() { }; rm.start(); - RMApp app = rm.submitApp(1024, true); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm) + .withUnmanaged(true) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt() .getAppAttemptId(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestSignalContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestSignalContainer.java index fac0b9662ef..e394499193a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestSignalContainer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestSignalContainer.java @@ -59,7 +59,7 @@ public void testSignalRequestDeliveryToNM() throws Exception { MockNM nm1 = rm.registerNode("h1:1234", 5000); - RMApp app = rm.submitApp(2000); + RMApp app = MockRMAppSubmitter.submitWithMemory(2000, rm); //kick the scheduling nm1.nodeHeartbeat(true); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestSubmitApplicationWithRMHA.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestSubmitApplicationWithRMHA.java index 4658d6030e4..f3e6beb1e4b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestSubmitApplicationWithRMHA.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestSubmitApplicationWithRMHA.java @@ -53,11 +53,18 @@ // submit the application with previous assigned applicationId // to current active rm: rm2 RMApp app1 = - rm2.submitApp(200, "", UserGroupInformation - .getCurrentUser().getShortUserName(), null, false, null, - configuration.getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, - false, false, true, appId); + MockRMAppSubmitter.submit(rm2, + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm2) + .withName("") + .withUser(UserGroupInformation + .getCurrentUser().getShortUserName()) + .withAcls(null) + .withUnmanaged(false) + .withQueue(null) + .withWaitForAccepted(false) + .withIsAppIdProvided(true) + .withApplicationId(appId) + .build()); // verify application submission verifySubmitApp(rm2, app1, appId); @@ -106,7 +113,7 @@ private void verifySubmitApp(MockRM rm, RMApp app, // Submit Application // After submission, the applicationState will be saved in RMStateStore. - RMApp app0 = rm1.submitApp(200); + RMApp app0 = MockRMAppSubmitter.submitWithMemory(200, rm1); // Do the failover explicitFailover(); @@ -137,12 +144,23 @@ private void verifySubmitApp(MockRM rm, RMApp app, // Submit Application // After submission, the applicationState will // not be saved in RMStateStore + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("") + .withUser(UserGroupInformation + .getCurrentUser().getShortUserName()) + .withAcls(null) + .withUnmanaged(false) + .withQueue(null) + .withMaxAppAttempts(configuration.getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, + YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS)) + .withCredentials(null) + .withAppType(null) + .withWaitForAccepted(false) + .withKeepContainers(false) + .build(); RMApp app0 = - rm1.submitApp(200, "", UserGroupInformation - .getCurrentUser().getShortUserName(), null, false, null, - configuration.getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, - false, false); + MockRMAppSubmitter.submit(rm1, data); // Do the failover explicitFailover(); @@ -163,11 +181,23 @@ private void verifySubmitApp(MockRM rm, RMApp app, // Application with previous applicationId // when catches the ApplicationNotFoundException RMApp app1 = - rm2.submitApp(200, "", UserGroupInformation - .getCurrentUser().getShortUserName(), null, false, null, - configuration.getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, - false, false, true, app0.getApplicationId()); + MockRMAppSubmitter.submit(rm2, + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm2) + .withName("") + .withUser(UserGroupInformation + .getCurrentUser().getShortUserName()) + .withAcls(null) + .withUnmanaged(false) + .withQueue(null) + .withMaxAppAttempts(configuration.getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, + YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS)) + .withCredentials(null) + .withAppType(null) + .withWaitForAccepted(false) + .withKeepContainers(false) + .withIsAppIdProvided(true) + .withApplicationId(app0.getApplicationId()) + .build()); verifySubmitApp(rm2, app1, app0.getApplicationId()); } @@ -183,7 +213,7 @@ public void testGetApplicationReportIdempotent() throws Exception{ // Submit Application // After submission, the applicationState will be saved in RMStateStore. - RMApp app = rm1.submitApp(200); + RMApp app = MockRMAppSubmitter.submitWithMemory(200, rm1); ApplicationReport appReport1 = rm1.getApplicationReport(app.getApplicationId()); @@ -235,7 +265,7 @@ public void testGetApplicationReportIdempotent() throws Exception{ // Submit Application // After submission, the applicationState will be saved in RMStateStore. - RMApp app0 = rm1.submitApp(200); + RMApp app0 = MockRMAppSubmitter.submitWithMemory(200, rm1); // Do the failover explicitFailover(); @@ -251,11 +281,23 @@ public void testGetApplicationReportIdempotent() throws Exception{ // check whether this application has been exist. If yes, just simply // return submitApplicationResponse. RMApp app1 = - rm2.submitApp(200, "", UserGroupInformation - .getCurrentUser().getShortUserName(), null, false, null, - configuration.getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, - false, false, true, app0.getApplicationId()); + MockRMAppSubmitter.submit(rm2, + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm2) + .withName("") + .withUser(UserGroupInformation + .getCurrentUser().getShortUserName()) + .withAcls(null) + .withUnmanaged(false) + .withQueue(null) + .withMaxAppAttempts(configuration.getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, + YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS)) + .withCredentials(null) + .withAppType(null) + .withWaitForAccepted(false) + .withKeepContainers(false) + .withIsAppIdProvided(true) + .withApplicationId(app0.getApplicationId()) + .build()); Assert.assertEquals(app1.getApplicationId(), app0.getApplicationId()); } @@ -273,12 +315,23 @@ public void testGetApplicationReportIdempotent() throws Exception{ // Submit Application // After submission, the applicationState will // not be saved in RMStateStore + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("") + .withUser(UserGroupInformation + .getCurrentUser().getShortUserName()) + .withAcls(null) + .withUnmanaged(false) + .withQueue(null) + .withMaxAppAttempts(configuration.getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, + YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS)) + .withCredentials(null) + .withAppType(null) + .withWaitForAccepted(false) + .withKeepContainers(false) + .build(); RMApp app0 = - rm1.submitApp(200, "", UserGroupInformation - .getCurrentUser().getShortUserName(), null, false, null, - configuration.getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, - false, false); + MockRMAppSubmitter.submit(rm1, data); // Do the failover explicitFailover(); @@ -294,11 +347,23 @@ public void testGetApplicationReportIdempotent() throws Exception{ // submitApplication() when failover happens during the submission process // because the submitApplication api is marked as idempotent RMApp app1 = - rm2.submitApp(200, "", UserGroupInformation - .getCurrentUser().getShortUserName(), null, false, null, - configuration.getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, - false, false, true, app0.getApplicationId()); + MockRMAppSubmitter.submit(rm2, + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm2) + .withName("") + .withUser(UserGroupInformation + .getCurrentUser().getShortUserName()) + .withAcls(null) + .withUnmanaged(false) + .withQueue(null) + .withMaxAppAttempts(configuration.getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, + YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS)) + .withCredentials(null) + .withAppType(null) + .withWaitForAccepted(false) + .withKeepContainers(false) + .withIsAppIdProvided(true) + .withApplicationId(app0.getApplicationId()) + .build()); verifySubmitApp(rm2, app1, app0.getApplicationId()); Assert.assertTrue(rm2.getRMContext().getRMApps() diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java index a821b0ad0a1..5c3d3d5b9da 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java @@ -171,7 +171,7 @@ public void testSchedulerRecovery() throws Exception { MockNM nm1 = new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService()); nm1.registerNode(); - RMApp app1 = rm1.submitApp(200); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(200, rm1); Resource amResources = app1.getAMResourceRequests().get(0).getCapability(); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); @@ -335,9 +335,15 @@ public void testDynamicQueueRecovery() throws Exception { // dynamic queue. rm1.getRMContext().getReservationSystem() .synchronizePlan(ReservationSystemTestUtil.reservationQ, true); - RMApp app1 = rm1.submitApp(200, "dynamicQApp", - UserGroupInformation.getCurrentUser().getShortUserName(), null, - ReservationSystemTestUtil.getReservationQueueName()); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("dynamicQApp") + .withUser(UserGroupInformation.getCurrentUser().getShortUserName()) + .withAcls(null) + .withQueue(ReservationSystemTestUtil.getReservationQueueName()) + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); Resource amResources = app1.getAMResourceRequests().get(0).getCapability(); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); @@ -617,7 +623,13 @@ public void testRMRestartWithRemovedQueue() throws Exception{ MockNM nm1 = new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService()); nm1.registerNode(); - final RMApp app1 = rm1.submitApp(1024, "app1", USER_1, null); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm1) + .withName("app1") + .withUser(USER_1) + .withAcls(null) + .build(); + final RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1,rm1, nm1); MockRM.finishAMAndVerifyAppState(app1, rm1, nm1, am1); @@ -670,12 +682,36 @@ public void testCapacitySchedulerRecovery() throws Exception { new MockNM("127.1.1.1:4321", 8192, rm1.getResourceTrackerService()); nm1.registerNode(); nm2.registerNode(); - RMApp app1_1 = rm1.submitApp(1024, "app1_1", USER_1, null, A); + MockRMAppSubmissionData data2 = + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm1) + .withName("app1_1") + .withUser(USER_1) + .withAcls(null) + .withQueue(A) + .withUnmanaged(false) + .build(); + RMApp app1_1 = MockRMAppSubmitter.submit(rm1, data2); MockAM am1_1 = MockRM.launchAndRegisterAM(app1_1, rm1, nm1); - RMApp app1_2 = rm1.submitApp(1024, "app1_2", USER_1, null, A); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm1) + .withName("app1_2") + .withUser(USER_1) + .withAcls(null) + .withQueue(A) + .withUnmanaged(false) + .build(); + RMApp app1_2 = MockRMAppSubmitter.submit(rm1, data1); MockAM am1_2 = MockRM.launchAndRegisterAM(app1_2, rm1, nm2); - RMApp app2 = rm1.submitApp(1024, "app2", USER_2, null, B); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm1) + .withName("app2") + .withUser(USER_2) + .withAcls(null) + .withQueue(B) + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm2); // clear queue metrics @@ -840,7 +876,15 @@ public void testCapacityLeafQueueBecomesParentOnRecovery() throws Exception { nm.registerNode(); // Submit an app to QueueB. - RMApp app = rm1.submitApp(1024, "app", USER_2, null, B); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm1) + .withName("app") + .withUser(USER_2) + .withAcls(null) + .withQueue(B) + .withUnmanaged(false) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm1, data); MockRM.launchAndRegisterAM(app, rm1, nm); assertEquals(rm1.getApplicationReport(app.getApplicationId()). getYarnApplicationState(), YarnApplicationState.RUNNING); @@ -888,20 +932,50 @@ public void testCapacitySchedulerQueueRemovedRecovery() throws Exception { new MockNM("127.1.1.1:4321", 8192, rm1.getResourceTrackerService()); nm1.registerNode(); nm2.registerNode(); - RMApp app1_1 = rm1.submitApp(1024, "app1_1", USER_1, null, A); + MockRMAppSubmissionData data2 = + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm1) + .withName("app1_1") + .withUser(USER_1) + .withAcls(null) + .withQueue(A) + .withUnmanaged(false) + .build(); + RMApp app1_1 = MockRMAppSubmitter.submit(rm1, data2); MockAM am1_1 = MockRM.launchAndRegisterAM(app1_1, rm1, nm1); - RMApp app1_2 = rm1.submitApp(1024, "app1_2", USER_1, null, A); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm1) + .withName("app1_2") + .withUser(USER_1) + .withAcls(null) + .withQueue(A) + .withUnmanaged(false) + .build(); + RMApp app1_2 = MockRMAppSubmitter.submit(rm1, data1); MockAM am1_2 = MockRM.launchAndRegisterAM(app1_2, rm1, nm2); - RMApp app2 = rm1.submitApp(1024, "app2", USER_2, null, B); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm1) + .withName("app2") + .withUser(USER_2) + .withAcls(null) + .withQueue(B) + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm2); assertEquals(rm1.getApplicationReport(app2.getApplicationId()). getYarnApplicationState(), YarnApplicationState.RUNNING); //Submit an app with a non existant queue to make sure it does not //cause a fatal failure in the non-recovery case - RMApp appNA = rm1.submitApp(1024, "app1_2", USER_1, null, - QUEUE_DOESNT_EXIST, false); + RMApp appNA = MockRMAppSubmitter.submit(rm1, + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm1) + .withName("app1_2") + .withUser(USER_1) + .withAcls(null) + .withQueue(QUEUE_DOESNT_EXIST) + .withWaitForAccepted(false) + .build()); // clear queue metrics rm1.clearQueueMetrics(app1_1); @@ -939,7 +1013,7 @@ public void testAMfailedBetweenRMRestart() throws Exception { MockNM nm1 = new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService()); nm1.registerNode(); - RMApp app1 = rm1.submitApp(200); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); rm2 = new MockRM(conf, rm1.getRMStateStore()); @@ -992,7 +1066,7 @@ public void testContainersNotRecoveredForCompletedApps() throws Exception { MockNM nm1 = new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService()); nm1.registerNode(); - RMApp app1 = rm1.submitApp(200); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); MockRM.finishAMAndVerifyAppState(app1, rm1, nm1, am1); @@ -1032,7 +1106,7 @@ public void testAppReregisterOnRMWorkPreservingRestart() throws Exception { nm1.registerNode(); // create app and launch the AM - RMApp app0 = rm1.submitApp(200); + RMApp app0 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am0 = MockRM.launchAM(app0, rm1, nm1); // Issuing registerAppAttempt() before and after RM restart to confirm // registerApplicationMaster() is idempotent. @@ -1059,7 +1133,7 @@ public void testAMContainerStatusWithRMRestart() throws Exception { MockNM nm1 = new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService()); nm1.registerNode(); - RMApp app1_1 = rm1.submitApp(1024); + RMApp app1_1 = MockRMAppSubmitter.submitWithMemory(1024, rm1); MockAM am1_1 = MockRM.launchAndRegisterAM(app1_1, rm1, nm1); RMAppAttempt attempt0 = app1_1.getCurrentAppAttempt(); @@ -1095,7 +1169,7 @@ public void testRecoverSchedulerAppAndAttemptSynchronously() throws Exception { nm1.registerNode(); // create app and launch the AM - RMApp app0 = rm1.submitApp(200); + RMApp app0 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am0 = MockRM.launchAndRegisterAM(app0, rm1, nm1); rm2 = new MockRM(conf, rm1.getRMStateStore()); @@ -1124,7 +1198,7 @@ public void testReleasedContainerNotRecovered() throws Exception { nm1.registerNode(); rm1.start(); - RMApp app1 = rm1.submitApp(1024); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(1024, rm1); final MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // Re-start RM @@ -1220,7 +1294,7 @@ public void testNewContainersNotAllocatedDuringSchedulerRecovery() MockNM nm1 = new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService()); nm1.registerNode(); - RMApp app1 = rm1.submitApp(200); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // Restart RM @@ -1278,7 +1352,7 @@ public void testRetriedFinishApplicationMasterRequest() nm1.registerNode(); // create app and launch the AM - RMApp app0 = rm1.submitApp(200); + RMApp app0 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am0 = MockRM.launchAM(app0, rm1, nm1); am0.registerAppAttempt(); @@ -1309,7 +1383,7 @@ public void testAppFailedToRenewTokenOnRecovery() throws Exception { MockNM nm1 = new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService()); nm1.registerNode(); - RMApp app1 = rm1.submitApp(200); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); MockRM rm2 = new TestSecurityMockRM(conf, rm1.getRMStateStore()) { @@ -1354,7 +1428,7 @@ public void testAppFailToValidateResourceRequestOnRecovery() throws Exception{ MockNM nm1 = new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService()); nm1.registerNode(); - RMApp app1 = rm1.submitApp(200); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // Change the config so that validateAndCreateResourceRequest throws @@ -1379,10 +1453,11 @@ public void testContainerCompleteMsgNotLostAfterAMFailedAndRMRestart() throws Ex // submit app with keepContainersAcrossApplicationAttempts true Resource resource = Records.newRecord(Resource.class); resource.setMemorySize(200); - RMApp app0 = rm1.submitApp(resource, "", UserGroupInformation - .getCurrentUser().getShortUserName(), null, false, null, - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS, null, null, true, true, - false, null, 0, null, true, null); + MockRMAppSubmissionData data = MockRMAppSubmissionData.Builder.createWithDefaults(resource, rm1) + .withKeepContainers(true) + .withMaxAppAttempts(YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS) + .build(); + RMApp app0 = MockRMAppSubmitter.submit(rm1, data); MockAM am0 = MockRM.launchAndRegisterAM(app0, rm1, nm1); am0.allocate("127.0.0.1", 1000, 2, new ArrayList()); @@ -1446,7 +1521,7 @@ public void testAppStateSavedButAttemptStateNotSaved() throws Exception { MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService()); nm1.registerNode(); - RMApp app1 = rm1.submitApp(200); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); MockRM.finishAMAndVerifyAppState(app1, rm1, nm1, am1); @@ -1482,7 +1557,11 @@ public void testUAMRecoveryOnRMWorkPreservingRestart() throws Exception { nm1.registerNode(); // create app and launch the UAM - RMApp app0 = rm1.submitApp(200, true); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withUnmanaged(true) + .build(); + RMApp app0 = MockRMAppSubmitter.submit(rm1, data); MockAM am0 = MockRM.launchUAM(app0, rm1, nm1); am0.registerAppAttempt(); @@ -1572,7 +1651,11 @@ public void testUnknownUserOnRecovery() throws Exception { nm1.registerNode(); // create app and launch the UAM - RMApp app0 = rm1.submitApp(200, true); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withUnmanaged(true) + .build(); + RMApp app0 = MockRMAppSubmitter.submit(rm1, data); MockAM am0 = MockRM.launchUAM(app0, rm1, nm1); am0.registerAppAttempt(); rm1.killApp(app0.getApplicationId()); @@ -1648,7 +1731,15 @@ public void testDynamicAutoCreatedQueueRecovery(String user, String queueName) rm1.getResourceTrackerService()); nm1.registerNode(); // 2. submit app to queue which is auto-created. - RMApp app1 = rm1.submitApp(200, "autoCreatedQApp", user, null, queueName); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("autoCreatedQApp") + .withUser(user) + .withAcls(null) + .withQueue(queueName) + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); Resource amResources = app1.getAMResourceRequests().get(0).getCapability(); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); @@ -1752,7 +1843,7 @@ public void testFairSchedulerCompletedAppsQueue() throws Exception { MockNM nm1 = new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService()); nm1.registerNode(); - RMApp app = rm1.submitApp(200); + RMApp app = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am1 = MockRM.launchAndRegisterAM(app, rm1, nm1); MockRM.finishAMAndVerifyAppState(app, rm1, nm1, am1); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingUnmanagedAM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingUnmanagedAM.java index 80f7319bdae..a03f42f5469 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingUnmanagedAM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingUnmanagedAM.java @@ -78,10 +78,20 @@ protected void testUAMRestart(boolean keepContainers) throws Exception { boolean unamanged = true; int maxAttempts = 1; boolean waitForAccepted = true; - RMApp app = rm.submitApp(200, "", - UserGroupInformation.getCurrentUser().getShortUserName(), null, - unamanged, null, maxAttempts, null, null, waitForAccepted, - keepContainers); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm) + .withName("") + .withUser(UserGroupInformation.getCurrentUser().getShortUserName()) + .withAcls(null) + .withUnmanaged(unamanged) + .withQueue(null) + .withMaxAppAttempts(maxAttempts) + .withCredentials(null) + .withAppType(null) + .withWaitForAccepted(waitForAccepted) + .withKeepContainers(keepContainers) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); MockAM am = MockRM.launchUAM(app, rm, nm); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java index b55012304cf..a77df99fb25 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java @@ -27,6 +27,7 @@ import java.util.Map; import java.util.Random; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.junit.Assert; import org.apache.hadoop.conf.Configuration; @@ -444,7 +445,7 @@ private void testRMWritingMassiveHistory(MockRM rm) throws Exception { rm.start(); MockNM nm = rm.registerNode("127.0.0.1:1234", 1024 * 10100); - RMApp app = rm.submitApp(1024); + RMApp app = MockRMAppSubmitter.submitWithMemory(1024, rm); //Wait to make sure the attempt has the right state //TODO explore a better way than sleeping for a while (YARN-4929) Thread.sleep(1000); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java index dd9bcd08b49..a14130f9fd0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java @@ -21,6 +21,7 @@ import java.security.PrivilegedExceptionAction; import java.util.List; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.junit.Assert; import org.apache.hadoop.conf.Configuration; @@ -112,7 +113,7 @@ public void testAMRMDecommissioningNodes() throws Exception { MockNM nm2 = rm.registerNode("127.0.0.2:1234", 10000); rm.drainEvents(); - RMApp app1 = rm.submitApp(2000); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(2000, rm); // Trigger the scheduling so the AM gets 'launched' on nm1 nm1.nodeHeartbeat(true); @@ -148,7 +149,7 @@ public void testAMRMUnusableNodes() throws Exception { MockNM nm4 = rm.registerNode("127.0.0.4:1234", 10000); rm.drainEvents(); - RMApp app1 = rm.submitApp(2000); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(2000, rm); // Trigger the scheduling so the AM gets 'launched' on nm1 nm1.nodeHeartbeat(true); @@ -208,7 +209,7 @@ public void testAMRMUnusableNodes() throws Exception { Assert.assertEquals(NodeUpdateType.NODE_UNUSABLE, nr.getNodeUpdateType()); // registering another AM gives it the complete failed list - RMApp app2 = rm.submitApp(2000); + RMApp app2 = MockRMAppSubmitter.submitWithMemory(2000, rm); // Trigger nm2 heartbeat so that AM gets launched on it nm2.nodeHeartbeat(true); RMAppAttempt attempt2 = app2.getCurrentAppAttempt(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCResponseId.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCResponseId.java index d609d7a7694..aac46f7caa2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCResponseId.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCResponseId.java @@ -30,6 +30,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.junit.After; @@ -77,7 +78,7 @@ public void testARRMResponseId() throws Exception { MockNM nm1 = rm.registerNode("h1:1234", 5000); - RMApp app = rm.submitApp(2000); + RMApp app = MockRMAppSubmitter.submitWithMemory(2000, rm); // Trigger the scheduling so the AM gets 'launched' nm1.nodeHeartbeat(true); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java index 9f122cb34bd..aa74bf5a620 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java @@ -27,6 +27,7 @@ import com.google.common.base.Throwables; import com.google.common.collect.Lists; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; @@ -46,6 +47,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; import org.apache.hadoop.yarn.server.resourcemanager.MockMemoryRMStateStore; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.ParameterizedSchedulerTestBase; import org.apache.hadoop.yarn.server.resourcemanager.TestRMRestart; import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore; @@ -80,10 +83,21 @@ public void testAMRestartWithExistingContainers() throws Exception { MockRM rm1 = new MockRM(getConf()); rm1.start(); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("name") + .withUser("user") + .withAcls(new HashMap()) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(-1) + .withCredentials(null) + .withAppType("MAPREDUCE") + .withWaitForAccepted(false) + .withKeepContainers(true) + .build(); RMApp app1 = - rm1.submitApp(200, "name", "user", - new HashMap(), false, "default", -1, - null, "MAPREDUCE", false, true); + MockRMAppSubmitter.submit(rm1, data); MockNM nm1 = new MockNM("127.0.0.1:1234", 10240, rm1.getResourceTrackerService()); nm1.registerNode(); @@ -283,10 +297,21 @@ public void testNMTokensRebindOnAMRestart() throws Exception { MockRM rm1 = new MockRM(getConf()); rm1.start(); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("myname") + .withUser("myuser") + .withAcls(new HashMap()) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(-1) + .withCredentials(null) + .withAppType("MAPREDUCE") + .withWaitForAccepted(false) + .withKeepContainers(true) + .build(); RMApp app1 = - rm1.submitApp(200, "myname", "myuser", - new HashMap(), false, "default", -1, - null, "MAPREDUCE", false, true); + MockRMAppSubmitter.submit(rm1, data); MockNM nm1 = new MockNM("127.0.0.1:1234", 8000, rm1.getResourceTrackerService()); nm1.registerNode(); @@ -394,7 +419,7 @@ public void testShouldNotCountFailureToMaxAttemptRetry() throws Exception { MockNM nm1 = new MockNM("127.0.0.1:1234", 8000, rm1.getResourceTrackerService()); nm1.registerNode(); - RMApp app1 = rm1.submitApp(200); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(200, rm1); RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); AbstractYarnScheduler scheduler = @@ -519,7 +544,7 @@ public void testMaxAttemptOneMeansOne() throws Exception { MockNM nm1 = new MockNM("127.0.0.1:1234", 8000, rm1.getResourceTrackerService()); nm1.registerNode(); - RMApp app1 = rm1.submitApp(200); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(200, rm1); RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); AbstractYarnScheduler scheduler = @@ -558,7 +583,7 @@ public void testPreemptedAMRestartOnRMRestart() throws Exception { MockNM nm1 = new MockNM("127.0.0.1:1234", 8000, rm1.getResourceTrackerService()); nm1.registerNode(); - RMApp app1 = rm1.submitApp(200); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(200, rm1); RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); AbstractYarnScheduler scheduler = @@ -647,7 +672,7 @@ public void testRMRestartOrFailoverNotCountedForAMFailures() MockNM nm1 = new MockNM("127.0.0.1:1234", 8000, rm1.getResourceTrackerService()); nm1.registerNode(); - RMApp app1 = rm1.submitApp(200); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(200, rm1); // AM should be restarted even though max-am-attempt is 1. MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); @@ -725,7 +750,13 @@ public void testRMAppAttemptFailuresValidityInterval() throws Exception { // set window size to a larger number : 60s // we will verify the app should be failed if // two continuous attempts failed in 60s. - RMApp app = rm1.submitApp(200, 60000, false); + RMApp app = MockRMAppSubmitter.submit(rm1, + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("") + .withUser(UserGroupInformation.getCurrentUser().getShortUserName()) + .withKeepContainers(false) + .withAttemptFailuresValidityInterval((long) 60000) + .build()); MockAM am = MockRM.launchAM(app, rm1, nm1); // Fail current attempt normally @@ -746,7 +777,13 @@ public void testRMAppAttemptFailuresValidityInterval() throws Exception { ControlledClock clock = new ControlledClock(); // set window size to 10s - RMAppImpl app1 = (RMAppImpl)rm1.submitApp(200, 10000, false); + RMAppImpl app1 = (RMAppImpl) MockRMAppSubmitter.submit(rm1, + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("") + .withUser(UserGroupInformation.getCurrentUser().getShortUserName()) + .withKeepContainers(false) + .withAttemptFailuresValidityInterval((long) 10000) + .build()); app1.setSystemClock(clock); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // Fail attempt1 normally @@ -857,10 +894,21 @@ public void testAMRestartNotLostContainerCompleteMsg() throws Exception { MockRM rm1 = new MockRM(getConf()); rm1.start(); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("name") + .withUser("user") + .withAcls(new HashMap()) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(-1) + .withCredentials(null) + .withAppType("MAPREDUCE") + .withWaitForAccepted(false) + .withKeepContainers(true) + .build(); RMApp app1 = - rm1.submitApp(200, "name", "user", - new HashMap(), false, "default", -1, - null, "MAPREDUCE", false, true); + MockRMAppSubmitter.submit(rm1, data); MockNM nm1 = new MockNM("127.0.0.1:1234", 10240, rm1.getResourceTrackerService()); nm1.registerNode(); @@ -960,7 +1008,13 @@ public void testAMRestartNotLostContainerAfterAttemptFailuresValidityInterval() nm1.registerNode(); // set window size to 10s and enable keepContainers - RMAppImpl app1 = (RMAppImpl)rm1.submitApp(200, 10000, true); + RMAppImpl app1 = (RMAppImpl) MockRMAppSubmitter.submit(rm1, + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("") + .withUser(UserGroupInformation.getCurrentUser().getShortUserName()) + .withKeepContainers(true) + .withAttemptFailuresValidityInterval((long) 10000) + .build()); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); int NUM_CONTAINERS = 2; allocateContainers(nm1, am1, NUM_CONTAINERS); @@ -1056,9 +1110,20 @@ public void testContainersFromPreviousAttemptsWithRMRestart() MockNM nm2 = new MockNM(nm2Address, 4089, rm1.getResourceTrackerService()); nm2.registerNode(); - RMApp app1 = rm1.submitApp(200, "name", "user", - new HashMap<>(), false, "default", -1, - null, "MAPREDUCE", false, true); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("name") + .withUser("user") + .withAcls(new HashMap<>()) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(-1) + .withCredentials(null) + .withAppType("MAPREDUCE") + .withWaitForAccepted(false) + .withKeepContainers(true) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); allocateContainers(nm1, am1, 1); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/TestResourceProfiles.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/TestResourceProfiles.java index 9db129f32af..e1fdf0a25e4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/TestResourceProfiles.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/TestResourceProfiles.java @@ -26,6 +26,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.junit.Assert; @@ -162,7 +163,7 @@ public void testResourceProfilesInAMResponse() throws Exception { MockRM rm = new MockRM(conf); rm.start(); MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * 1024); - RMApp app1 = rm.submitApp(2048); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(2048, rm); nm1.nodeHeartbeat(true); RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId()); @@ -175,7 +176,7 @@ public void testResourceProfilesInAMResponse() throws Exception { rm = new MockRM(conf); rm.start(); nm1 = rm.registerNode("127.0.0.1:1234", 6 * 1024); - app1 = rm.submitApp(2048); + app1 = MockRMAppSubmitter.submitWithMemory(2048, rm); nm1.nodeHeartbeat(true); attempt1 = app1.getCurrentAppAttempt(); am1 = rm.sendAMLaunched(attempt1.getAppAttemptId()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestApplicationLifetimeMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestApplicationLifetimeMonitor.java index e1d87163b26..f3d6e73c26d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestApplicationLifetimeMonitor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestApplicationLifetimeMonitor.java @@ -48,6 +48,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.TestRMRestart; import org.apache.hadoop.yarn.server.resourcemanager.TestWorkPreservingRMRestart; import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore; @@ -127,18 +129,34 @@ public void testApplicationLifetimeMonitor() Map timeouts = new HashMap(); timeouts.put(ApplicationTimeoutType.LIFETIME, 10L); - RMApp app1 = rm.submitApp(1024, appPriority, timeouts); + RMApp app1 = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm) + .withPriority(appPriority) + .withApplicationTimeouts(timeouts) + .build()); // 20L seconds timeouts.put(ApplicationTimeoutType.LIFETIME, 20L); - RMApp app2 = rm.submitApp(1024, appPriority, timeouts); + RMApp app2 = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm) + .withPriority(appPriority) + .withApplicationTimeouts(timeouts) + .build()); // user not set lifetime, so queue max lifetime will be considered. - RMApp app3 = rm.submitApp(1024, appPriority, Collections.emptyMap()); + RMApp app3 = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm) + .withPriority(appPriority) + .withApplicationTimeouts(Collections.emptyMap()) + .build()); // asc lifetime exceeds queue max lifetime timeouts.put(ApplicationTimeoutType.LIFETIME, 40L); - RMApp app4 = rm.submitApp(1024, appPriority, timeouts); + RMApp app4 = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm) + .withPriority(appPriority) + .withApplicationTimeouts(timeouts) + .build()); nm1.nodeHeartbeat(true); // Send launch Event @@ -238,7 +256,11 @@ public void testApplicationLifetimeOnRMRestart() throws Exception { Map timeouts = new HashMap(); timeouts.put(ApplicationTimeoutType.LIFETIME, appLifetime); - RMApp app1 = rm1.submitApp(200, Priority.newInstance(0), timeouts); + RMApp app1 = MockRMAppSubmitter.submit(rm1, + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withPriority(Priority.newInstance(0)) + .withApplicationTimeouts(timeouts) + .build()); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // Re-start RM @@ -322,7 +344,11 @@ public synchronized void updateApplicationStateInternal( Map timeouts = new HashMap(); timeouts.put(ApplicationTimeoutType.LIFETIME, appLifetime); - RMApp app1 = rm1.submitApp(200, Priority.newInstance(0), timeouts); + RMApp app1 = MockRMAppSubmitter.submit(rm1, + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withPriority(Priority.newInstance(0)) + .withApplicationTimeouts(timeouts) + .build()); Map updateTimeout = new HashMap(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestNodesListManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestNodesListManager.java index e41bbc80076..b91b03900ba 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestNodesListManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestNodesListManager.java @@ -37,6 +37,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockNodes; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.NodesListManager; import org.apache.hadoop.yarn.server.resourcemanager.NodesListManagerEvent; import org.apache.hadoop.yarn.server.resourcemanager.NodesListManagerEventType; @@ -74,12 +75,12 @@ protected Dispatcher createDispatcher() { RMNode rmnode = MockNodes.newNodeInfo(1, clusterResource); // Create killing APP - RMApp killrmApp = rm.submitApp(200); + RMApp killrmApp = MockRMAppSubmitter.submitWithMemory(200, rm); rm.killApp(killrmApp.getApplicationId()); rm.waitForState(killrmApp.getApplicationId(), RMAppState.KILLED); // Create finish APP - RMApp finshrmApp = rm.submitApp(2000); + RMApp finshrmApp = MockRMAppSubmitter.submitWithMemory(2000, rm); nm1.nodeHeartbeat(true); RMAppAttempt attempt = finshrmApp.getCurrentAppAttempt(); MockAM am = rm.sendAMLaunched(attempt.getAppAttemptId()); @@ -89,7 +90,7 @@ protected Dispatcher createDispatcher() { rm.waitForState(am.getApplicationAttemptId(), RMAppAttemptState.FINISHED); // Create submitted App - RMApp subrmApp = rm.submitApp(200); + RMApp subrmApp = MockRMAppSubmitter.submitWithMemory(200, rm); // Fire Event for NODE_USABLE nodesListManager.handle(new NodesListManagerEvent( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java index bb6591b3701..3cfae3a1703 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java @@ -53,6 +53,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter; import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher; @@ -283,7 +284,7 @@ public void testExistenceOfResourceRequestInRMContainer() throws Exception { MockRM rm1 = new MockRM(conf); rm1.start(); MockNM nm1 = rm1.registerNode("unknownhost:1234", 8000); - RMApp app1 = rm1.submitApp(1024); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(1024, rm1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); ResourceScheduler scheduler = rm1.getResourceScheduler(); @@ -323,7 +324,7 @@ public void testStoreAllContainerMetrics() throws Exception { rm1.start(); MockNM nm1 = rm1.registerNode("unknownhost:1234", 8000); - RMApp app1 = rm1.submitApp(1024); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(1024, rm1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); nm1.nodeHeartbeat(am1.getApplicationAttemptId(), 1, ContainerState.RUNNING); @@ -363,7 +364,7 @@ public void testStoreOnlyAMContainerMetrics() throws Exception { rm1.start(); MockNM nm1 = rm1.registerNode("unknownhost:1234", 8000); - RMApp app1 = rm1.submitApp(1024); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(1024, rm1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); nm1.nodeHeartbeat(am1.getApplicationAttemptId(), 1, ContainerState.RUNNING); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java index ba409b1386b..6ba0843754a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java @@ -50,6 +50,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockNodes; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.NMLivelinessMonitor; import org.apache.hadoop.yarn.server.resourcemanager.NodesListManager; import org.apache.hadoop.yarn.server.resourcemanager.ParameterizedSchedulerTestBase; @@ -399,7 +401,7 @@ public void testReleasedContainerIfAppAttemptisNull() throws Exception { new SchedulerApplication(null, mockAPp.getUser()); // Second app with one app attempt - RMApp app = rm1.submitApp(200); + RMApp app = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am1 = MockRM.launchAndRegisterAM(app, rm1, nm1); final ContainerId runningContainer = ContainerId.newContainerId(am1.getApplicationAttemptId(), 2); @@ -436,9 +438,21 @@ public void testContainerReleaseWithAllocationTags() throws Exception { MockNM nm1 = new MockNM("127.0.0.1:1234", 10240, rm1.getResourceTrackerService()); nm1.registerNode(); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("name") + .withUser("user") + .withAcls(new HashMap<>()) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(-1) + .withCredentials(null) + .withAppType("Test") + .withWaitForAccepted(false) + .withKeepContainers(true) + .build(); RMApp app1 = - rm1.submitApp(200, "name", "user", new HashMap<>(), false, "default", - -1, null, "Test", false, true); + MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // allocate 1 container with tag1 @@ -527,10 +541,21 @@ public void testContainerReleasedByNode() throws Exception { MockRM rm1 = new MockRM(conf); try { rm1.start(); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("name") + .withUser("user") + .withAcls(new HashMap()) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(-1) + .withCredentials(null) + .withAppType("Test") + .withWaitForAccepted(false) + .withKeepContainers(true) + .build(); RMApp app1 = - rm1.submitApp(200, "name", "user", - new HashMap(), false, "default", - -1, null, "Test", false, true); + MockRMAppSubmitter.submit(rm1, data); MockNM nm1 = new MockNM("127.0.0.1:1234", 10240, rm1.getResourceTrackerService()); nm1.registerNode(); @@ -639,10 +664,21 @@ public void testResourceRequestRestoreWhenRMContainerIsAtAllocated() MockRM rm1 = new MockRM(conf); try { rm1.start(); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("name") + .withUser("user") + .withAcls(new HashMap()) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(-1) + .withCredentials(null) + .withAppType("Test") + .withWaitForAccepted(false) + .withKeepContainers(true) + .build(); RMApp app1 = - rm1.submitApp(200, "name", "user", - new HashMap(), false, "default", - -1, null, "Test", false, true); + MockRMAppSubmitter.submit(rm1, data); MockNM nm1 = new MockNM("127.0.0.1:1234", 10240, rm1.getResourceTrackerService()); nm1.registerNode(); @@ -727,10 +763,21 @@ public void testResourceRequestRecoveryToTheRightAppAttempt() MockRM rm = new MockRM(conf); try { rm.start(); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm) + .withName("name") + .withUser("user") + .withAcls(new HashMap()) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(-1) + .withCredentials(null) + .withAppType("Test") + .withWaitForAccepted(false) + .withKeepContainers(true) + .build(); RMApp rmApp = - rm.submitApp(200, "name", "user", - new HashMap(), false, "default", -1, - null, "Test", false, true); + MockRMAppSubmitter.submit(rm, data); MockNM node = new MockNM("127.0.0.1:1234", 10240, rm.getResourceTrackerService()); node.registerNode(); @@ -985,10 +1032,21 @@ public void testContainerRecoveredByNode() throws Exception { MockRM rm1 = new MockRM(conf); try { rm1.start(); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("name") + .withUser("user") + .withAcls(new HashMap()) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(-1) + .withCredentials(null) + .withAppType("Test") + .withWaitForAccepted(false) + .withKeepContainers(true) + .build(); RMApp app1 = - rm1.submitApp(200, "name", "user", - new HashMap(), false, "default", - -1, null, "Test", false, true); + MockRMAppSubmitter.submit(rm1, data); MockNM nm1 = new MockNM("127.0.0.1:1234", 10240, rm1.getResourceTrackerService()); nm1.registerNode(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java index 696924203de..8a2c3ebcc8f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java @@ -79,6 +79,8 @@ import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; import org.apache.hadoop.yarn.resourcetypes.ResourceTypesTestHelper; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MockRMWithAMS; import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MyContainerManager; @@ -722,7 +724,13 @@ public void testValidateResourceBlacklistRequest() throws Exception { Map acls = new HashMap(2); acls.put(ApplicationAccessType.VIEW_APP, "*"); - RMApp app = rm.submitApp(1024, "appname", "appuser", acls); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm) + .withName("appname") + .withUser("appuser") + .withAcls(acls) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); nm1.nodeHeartbeat(true); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulingWithAllocationRequestId.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulingWithAllocationRequestId.java index 8e983321136..ce6342f1f71 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulingWithAllocationRequestId.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulingWithAllocationRequestId.java @@ -28,6 +28,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.ParameterizedSchedulerTestBase; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; @@ -72,7 +74,7 @@ public void testMultipleAllocationRequestIds() throws Exception { MockNM nm1 = rm.registerNode("127.0.0.1:1234", 4 * GB); MockNM nm2 = rm.registerNode("127.0.0.2:5678", 4 * GB); - RMApp app1 = rm.submitApp(2048); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(2048, rm); // kick the scheduling nm1.nodeHeartbeat(true); RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); @@ -114,7 +116,7 @@ public void testMultipleAllocationRequestDiffPriority() throws Exception { MockNM nm1 = rm.registerNode("127.0.0.1:1234", 4 * GB); MockNM nm2 = rm.registerNode("127.0.0.2:5678", 4 * GB); - RMApp app1 = rm.submitApp(2048); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(2048, rm); // kick the scheduling nm1.nodeHeartbeat(true); RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); @@ -175,7 +177,12 @@ public void testMultipleAppsWithAllocationReqId() throws Exception { nm2.registerNode(); // submit 1st app - RMApp app1 = rm.submitApp(1 * GB, "user_0", "a1"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("user_0") + .withUser("a1") + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); // Submit app1 RR with allocationReqId = 5 @@ -190,7 +197,12 @@ public void testMultipleAppsWithAllocationReqId() throws Exception { checkAllocatedContainer(allocated.get(0), 1 * GB, nm1.getNodeId(), 5L); // Submit another application - RMApp app2 = rm.submitApp(1 * GB, "user_1", "a2"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("user_1") + .withUser("a2") + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm2); // Submit app2 RR with allocationReqId = 5 diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java index 24ae244a969..dd9f3b80bf9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java @@ -53,6 +53,8 @@ import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; @@ -819,20 +821,41 @@ public RMNodeLabelsManager createNodeLabelManager() { // Submit application to queue c where the default partition capacity is // zero - RMApp app1 = rm.submitApp(GB, "app", "user", null, "c", false); + RMApp app1 = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c") + .withWaitForAccepted(false) + .build()); rm.drainEvents(); rm.waitForState(app1.getApplicationId(), RMAppState.ACCEPTED); assertEquals(RMAppState.ACCEPTED, app1.getState()); rm.killApp(app1.getApplicationId()); - RMApp app2 = rm.submitApp(GB, "app", "user", null, "a1", false); + RMApp app2 = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withWaitForAccepted(false) + .build()); rm.drainEvents(); rm.waitForState(app2.getApplicationId(), RMAppState.ACCEPTED); assertEquals(RMAppState.ACCEPTED, app2.getState()); // Check second application is rejected and based on queue level max // application app is rejected - RMApp app3 = rm.submitApp(GB, "app", "user", null, "a1", false); + RMApp app3 = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withWaitForAccepted(false) + .build()); rm.drainEvents(); rm.waitForState(app3.getApplicationId(), RMAppState.FAILED); assertEquals(RMAppState.FAILED, app3.getState()); @@ -843,15 +866,36 @@ public RMNodeLabelsManager createNodeLabelManager() { app3.getDiagnostics().toString()); // based on Global limit of queue usert application is rejected - RMApp app11 = rm.submitApp(GB, "app", "user", null, "d", false); + RMApp app11 = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("d") + .withWaitForAccepted(false) + .build()); rm.drainEvents(); rm.waitForState(app11.getApplicationId(), RMAppState.ACCEPTED); assertEquals(RMAppState.ACCEPTED, app11.getState()); - RMApp app12 = rm.submitApp(GB, "app", "user", null, "d", false); + RMApp app12 = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("d") + .withWaitForAccepted(false) + .build()); rm.drainEvents(); rm.waitForState(app12.getApplicationId(), RMAppState.ACCEPTED); assertEquals(RMAppState.ACCEPTED, app12.getState()); - RMApp app13 = rm.submitApp(GB, "app", "user", null, "d", false); + RMApp app13 = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("d") + .withWaitForAccepted(false) + .build()); rm.drainEvents(); rm.waitForState(app13.getApplicationId(), RMAppState.FAILED); assertEquals(RMAppState.FAILED, app13.getState()); @@ -862,10 +906,24 @@ public RMNodeLabelsManager createNodeLabelManager() { app13.getDiagnostics().toString()); // based on system max limit application is rejected - RMApp app14 = rm.submitApp(GB, "app", "user2", null, "a2", false); + RMApp app14 = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("app") + .withUser("user2") + .withAcls(null) + .withQueue("a2") + .withWaitForAccepted(false) + .build()); rm.drainEvents(); rm.waitForState(app14.getApplicationId(), RMAppState.ACCEPTED); - RMApp app15 = rm.submitApp(GB, "app", "user2", null, "a2", false); + RMApp app15 = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("app") + .withUser("user2") + .withAcls(null) + .withQueue("a2") + .withWaitForAccepted(false) + .build()); rm.drainEvents(); rm.waitForState(app15.getApplicationId(), RMAppState.FAILED); assertEquals(RMAppState.FAILED, app15.getState()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java index d73f1c84373..33ab4b8cd83 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java @@ -45,6 +45,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; @@ -150,14 +152,38 @@ public RMNodeLabelsManager createNodeLabelManager() { MockNM nm3 = rm1.registerNode("h3:1234", 10 * GB); // label = // Submit app1 with 1Gb AM resource to Queue A1 for label X - RMApp app1 = rm1.submitApp(GB, "app", "user", null, "a1", "x"); + MockRMAppSubmissionData data5 = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withAmLabel("x") + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data5); // Submit app2 with 1Gb AM resource to Queue A1 for label X - RMApp app2 = rm1.submitApp(GB, "app", "user", null, "a1", "x"); + MockRMAppSubmissionData data4 = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withAmLabel("x") + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data4); // Submit 3rd app to Queue A1 for label X, and this will be pending as // AM limit is already crossed for label X. (2GB) - RMApp pendingApp = rm1.submitApp(GB, "app", "user", null, "a1", "x"); + MockRMAppSubmissionData data3 = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withAmLabel("x") + .build(); + RMApp pendingApp = MockRMAppSubmitter.submit(rm1, data3); CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler(); LeafQueue leafQueue = (LeafQueue) cs.getQueue("a1"); @@ -180,16 +206,40 @@ public RMNodeLabelsManager createNodeLabelManager() { // Now verify the same test case in Queue C1 where label is not configured. // Submit an app to Queue C1 with empty label - RMApp app3 = rm1.submitApp(GB, "app", "user", null, "c1"); + MockRMAppSubmissionData data2 = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c1") + .withUnmanaged(false) + .build(); + RMApp app3 = MockRMAppSubmitter.submit(rm1, data2); MockRM.launchAndRegisterAM(app3, rm1, nm3); // Submit next app to Queue C1 with empty label - RMApp app4 = rm1.submitApp(GB, "app", "user", null, "c1"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c1") + .withUnmanaged(false) + .build(); + RMApp app4 = MockRMAppSubmitter.submit(rm1, data1); MockRM.launchAndRegisterAM(app4, rm1, nm3); // Submit 3rd app to Queue C1. This will be pending as Queue's am-limit // is reached. - pendingApp = rm1.submitApp(GB, "app", "user", null, "c1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c1") + .withUnmanaged(false) + .build(); + pendingApp = MockRMAppSubmitter.submit(rm1, data); leafQueue = (LeafQueue) cs.getQueue("c1"); Assert.assertNotNull(leafQueue); @@ -257,14 +307,30 @@ public RMNodeLabelsManager createNodeLabelManager() { MockNM nm3 = rm1.registerNode("h3:1234", 10 * GB); // label = // Submit app1 (2 GB) to Queue A1 and label X - RMApp app1 = rm1.submitApp(2 * GB, "app", "user", null, "a1", "x"); + MockRMAppSubmissionData data3 = + MockRMAppSubmissionData.Builder.createWithDefaults(2 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withAmLabel("x") + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data3); // This app must be activated eventhough the am-resource per-partition // limit is only for 1.5GB. MockRM.launchAndRegisterAM(app1, rm1, nm1); // Submit 2nd app to label "X" with one GB and it must be pending since // am-resource per-partition limit is crossed (1.5 GB was the limit). - rm1.submitApp(GB, "app", "user", null, "a1", "x"); + MockRMAppSubmissionData data2 = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withAmLabel("x") + .build(); + MockRMAppSubmitter.submit(rm1, data2); CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler(); LeafQueue leafQueue = (LeafQueue) cs.getQueue("a1"); @@ -278,14 +344,30 @@ public RMNodeLabelsManager createNodeLabelManager() { // to see queue level am-resource-limit is still working as expected. // Submit an app to Queue C1 with empty label (2 GB) - RMApp app3 = rm1.submitApp(2 * GB, "app", "user", null, "c1"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(2 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c1") + .withUnmanaged(false) + .build(); + RMApp app3 = MockRMAppSubmitter.submit(rm1, data1); // This app must be activated even though the am-resource per-queue // limit is only for 1.5GB MockRM.launchAndRegisterAM(app3, rm1, nm3); // Submit 2nd app to C1 (Default label, hence am-limit per-queue will be // considered). - rm1.submitApp(GB, "app", "user", null, "c1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c1") + .withUnmanaged(false) + .build(); + MockRMAppSubmitter.submit(rm1, data); leafQueue = (LeafQueue) cs.getQueue("c1"); Assert.assertNotNull(leafQueue); @@ -332,11 +414,27 @@ public RMNodeLabelsManager createNodeLabelManager() { rm1.registerNode("h3:1234", 10 * GB); // label = // Submit app1 (2 GB) to Queue A1 and label X - RMApp app1 = rm1.submitApp(2 * GB, "app", "user", null, "a1", "x"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(2 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withAmLabel("x") + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data1); // Submit 2nd app to label "X" with one GB. Since queue am-limit is 2GB, // 2nd app will be pending and first one will get activated. - RMApp pendingApp = rm1.submitApp(GB, "app", "user", null, "a1", "x"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withAmLabel("x") + .build(); + RMApp pendingApp = MockRMAppSubmitter.submit(rm1, data); CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler(); LeafQueue leafQueue = (LeafQueue) cs.getQueue("a1"); @@ -400,14 +498,30 @@ public RMNodeLabelsManager createNodeLabelManager() { rm1.registerNode("h3:1234", 10 * GB); // label = // Submit app1 with 1Gb AM resource to Queue A1 for label X for user0 - RMApp app1 = rm1.submitApp(GB, "app", user_0, null, "a1", "x"); + MockRMAppSubmissionData data3 = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm1) + .withName("app") + .withUser(user_0) + .withAcls(null) + .withQueue("a1") + .withAmLabel("x") + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data3); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // Place few allocate requests to make it an active application am1.allocate("*", 1 * GB, 15, new ArrayList(), ""); // Now submit 2nd app to Queue A1 for label X for user1 - RMApp app2 = rm1.submitApp(GB, "app", user_1, null, "a1", "x"); + MockRMAppSubmissionData data2 = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm1) + .withName("app") + .withUser(user_1) + .withAcls(null) + .withQueue("a1") + .withAmLabel("x") + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data2); MockRM.launchAndRegisterAM(app2, rm1, nm1); CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler(); @@ -421,7 +535,15 @@ public RMNodeLabelsManager createNodeLabelManager() { // Submit 3rd app to Queue A1 for label X for user1. Now user1 will have // 2 applications (2 GB resource) and user0 will have one app (1GB). - RMApp app3 = rm1.submitApp(GB, "app", user_1, null, "a1", "x"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm1) + .withName("app") + .withUser(user_1) + .withAcls(null) + .withQueue("a1") + .withAmLabel("x") + .build(); + RMApp app3 = MockRMAppSubmitter.submit(rm1, data1); MockAM am2 = MockRM.launchAndRegisterAM(app3, rm1, nm1); // Place few allocate requests to make it an active application. This is @@ -433,7 +555,15 @@ public RMNodeLabelsManager createNodeLabelManager() { // 4Gb -> 40% of label "X" in queue A1 // Since we have 2 users, 50% of 4Gb will be max for each user. Here user1 // has already crossed this 2GB limit, hence this app will be pending. - RMApp pendingApp = rm1.submitApp(GB, "app", user_1, null, "a1", "x"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm1) + .withName("app") + .withUser(user_1) + .withAcls(null) + .withQueue("a1") + .withAmLabel("x") + .build(); + RMApp pendingApp = MockRMAppSubmitter.submit(rm1, data); // Verify active applications count per user and also in queue level. Assert.assertEquals(3, leafQueue.getNumActiveApplications()); @@ -513,15 +643,39 @@ public RMNodeLabelsManager createNodeLabelManager() { MockNM nm5 = rm1.registerNode("h5:1234", 10 * GB); // label = // Submit app1 with 2Gb AM resource to Queue A1 for label Y - RMApp app1 = rm1.submitApp(2 * GB, "app", "user", null, "a1", "y"); + MockRMAppSubmissionData data4 = + MockRMAppSubmissionData.Builder.createWithDefaults(2 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withAmLabel("y") + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data4); MockRM.launchAndRegisterAM(app1, rm1, nm2); // Submit app2 with 1Gb AM resource to Queue A1 for label Y - RMApp app2 = rm1.submitApp(GB, "app", "user", null, "a1", "y"); + MockRMAppSubmissionData data3 = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withAmLabel("y") + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data3); MockRM.launchAndRegisterAM(app2, rm1, nm3); // Submit another app with 1Gb AM resource to Queue A1 for label Y - rm1.submitApp(GB, "app", "user", null, "a1", "y"); + MockRMAppSubmissionData data2 = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withAmLabel("y") + .build(); + MockRMAppSubmitter.submit(rm1, data2); CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler(); LeafQueue leafQueue = (LeafQueue) cs.getQueue("a1"); @@ -541,11 +695,27 @@ public RMNodeLabelsManager createNodeLabelManager() { Assert.assertEquals(1, leafQueue.getNumPendingApplications()); // Submit app3 with 1Gb AM resource to Queue B1 (no_label) - RMApp app3 = rm1.submitApp(GB, "app", "user", null, "b1"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + RMApp app3 = MockRMAppSubmitter.submit(rm1, data1); MockRM.launchAndRegisterAM(app3, rm1, nm5); // Submit another app with 1Gb AM resource to Queue B1 (no_label) - rm1.submitApp(GB, "app", "user", null, "b1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + MockRMAppSubmitter.submit(rm1, data); leafQueue = (LeafQueue) cs.getQueue("b1"); Assert.assertNotNull(leafQueue); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java index cad0151cf05..a02ce624846 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java @@ -40,6 +40,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockNodes; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl; @@ -160,7 +162,10 @@ public void testApplicationPriorityAllocation() throws Exception { Priority appPriority1 = Priority.newInstance(5); MockNM nm1 = rm.registerNode("127.0.0.1:1234", 16 * GB); - RMApp app1 = rm.submitApp(1 * GB, appPriority1); + MockRMAppSubmissionData data1 = MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withPriority(appPriority1) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data1); // kick the scheduler, 1 GB given to AM1, remaining 15GB on nm1 MockAM am1 = MockRM.launchAM(app1, rm, nm1); @@ -181,7 +186,10 @@ public void testApplicationPriorityAllocation() throws Exception { // Submit the second app App2 with priority 8 (Higher than App1) Priority appPriority2 = Priority.newInstance(8); - RMApp app2 = rm.submitApp(1 * GB, appPriority2); + MockRMAppSubmissionData data = MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withPriority(appPriority2) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm, data); // kick the scheduler, 1 GB which was free is given to AM of App2 MockAM am2 = MockRM.launchAM(app2, rm, nm1); @@ -245,7 +253,10 @@ public void testPriorityWithPendingApplications() throws Exception { Priority appPriority1 = Priority.newInstance(5); MockNM nm1 = rm.registerNode("127.0.0.1:1234", 8 * GB); - RMApp app1 = rm.submitApp(1 * GB, appPriority1); + MockRMAppSubmissionData data3 = MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withPriority(appPriority1) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data3); // kick the scheduler, 1 GB given to AM1, remaining 7GB on nm1 MockAM am1 = MockRM.launchAM(app1, rm, nm1); @@ -266,15 +277,24 @@ public void testPriorityWithPendingApplications() throws Exception { // Submit the second app App2 with priority 7 Priority appPriority2 = Priority.newInstance(7); - RMApp app2 = rm.submitApp(1 * GB, appPriority2); + MockRMAppSubmissionData data2 = MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withPriority(appPriority2) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm, data2); // Submit the third app App3 with priority 8 Priority appPriority3 = Priority.newInstance(8); - RMApp app3 = rm.submitApp(1 * GB, appPriority3); + MockRMAppSubmissionData data1 = MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withPriority(appPriority3) + .build(); + RMApp app3 = MockRMAppSubmitter.submit(rm, data1); // Submit the second app App4 with priority 6 Priority appPriority4 = Priority.newInstance(6); - RMApp app4 = rm.submitApp(1 * GB, appPriority4); + MockRMAppSubmissionData data = MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withPriority(appPriority4) + .build(); + RMApp app4 = MockRMAppSubmitter.submit(rm, data); // Only one app can run as AM resource limit restricts it. Kill app1, // If app3 (highest priority among rest) gets active, it indicates that @@ -308,7 +328,10 @@ public void testMaxPriorityValidation() throws Exception { Priority appPriority1 = Priority.newInstance(15); rm.registerNode("127.0.0.1:1234", 8 * GB); - RMApp app1 = rm.submitApp(1 * GB, appPriority1); + MockRMAppSubmissionData data = MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withPriority(appPriority1) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); // Application submission should be successful and verify priority Assert.assertEquals(app1.getApplicationSubmissionContext().getPriority(), @@ -329,7 +352,10 @@ public void testUpdatePriorityAtRuntime() throws Exception { Priority appPriority1 = Priority.newInstance(5); MockNM nm1 = rm.registerNode("127.0.0.1:1234", 16 * GB); - RMApp app1 = rm.submitApp(1 * GB, appPriority1); + MockRMAppSubmissionData data = MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withPriority(appPriority1) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); // kick the scheduler, 1 GB given to AM1, remaining 15GB on nm1 MockAM am1 = MockRM.launchAM(app1, rm, nm1); @@ -366,7 +392,10 @@ public void testUpdateInvalidPriorityAtRuntime() throws Exception { Priority appPriority1 = Priority.newInstance(5); MockNM nm1 = rm.registerNode("127.0.0.1:1234", 16 * GB); - RMApp app1 = rm.submitApp(1 * GB, appPriority1); + MockRMAppSubmissionData data = MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withPriority(appPriority1) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); // kick the scheduler, 1 GB given to AM1, remaining 15GB on nm1 MockAM am1 = MockRM.launchAM(app1, rm, nm1); @@ -414,7 +443,10 @@ public void testRMRestartWithChangeInPriority() throws Exception { nm1.registerNode(); Priority appPriority1 = Priority.newInstance(5); - RMApp app1 = rm1.submitApp(1 * GB, appPriority1); + MockRMAppSubmissionData data = MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withPriority(appPriority1) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); // kick the scheduler, 1 GB given to AM1, remaining 15GB on nm1 MockAM am1 = MockRM.launchAM(app1, rm1, nm1); @@ -469,7 +501,10 @@ public void testApplicationPriorityAllocationWithChangeInPriority() Priority appPriority1 = Priority.newInstance(5); MockNM nm1 = rm.registerNode("127.0.0.1:1234", 16 * GB); - RMApp app1 = rm.submitApp(1 * GB, appPriority1); + MockRMAppSubmissionData data1 = MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withPriority(appPriority1) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data1); // kick the scheduler, 1 GB given to AM1, remaining 15GB on nm1 MockAM am1 = MockRM.launchAM(app1, rm, nm1); @@ -491,7 +526,10 @@ public void testApplicationPriorityAllocationWithChangeInPriority() // Submit the second app App2 with priority 8 (Higher than App1) Priority appPriority2 = Priority.newInstance(8); - RMApp app2 = rm.submitApp(1 * GB, appPriority2); + MockRMAppSubmissionData data = MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withPriority(appPriority2) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm, data); // kick the scheduler, 1 GB which was free is given to AM of App2 MockAM am2 = MockRM.launchAM(app2, rm, nm1); @@ -619,13 +657,19 @@ public void testOrderOfActivatingThePriorityApplicationOnRMRestart() // App-1 with priority 5 submitted and running Priority appPriority1 = Priority.newInstance(5); - RMApp app1 = rm1.submitApp(memory, appPriority1); + MockRMAppSubmissionData data2 = MockRMAppSubmissionData.Builder.createWithDefaults(memory, rm1) + .withPriority(appPriority1) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data2); MockAM am1 = MockRM.launchAM(app1, rm1, nm1); am1.registerAppAttempt(); // App-2 with priority 6 submitted and running Priority appPriority2 = Priority.newInstance(6); - RMApp app2 = rm1.submitApp(memory, appPriority2); + MockRMAppSubmissionData data1 = MockRMAppSubmissionData.Builder.createWithDefaults(memory, rm1) + .withPriority(appPriority2) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data1); MockAM am2 = MockRM.launchAM(app2, rm1, nm1); am2.registerAppAttempt(); @@ -636,7 +680,10 @@ public void testOrderOfActivatingThePriorityApplicationOnRMRestart() // App-3 with priority 7 submitted and scheduled. But not activated since // AMResourceLimit threshold Priority appPriority3 = Priority.newInstance(7); - RMApp app3 = rm1.submitApp(memory, appPriority3); + MockRMAppSubmissionData data = MockRMAppSubmissionData.Builder.createWithDefaults(memory, rm1) + .withPriority(appPriority3) + .build(); + RMApp app3 = MockRMAppSubmitter.submit(rm1, data); rm1.drainEvents(); Assert.assertEquals(2, defaultQueue.getNumActiveApplications()); @@ -739,7 +786,10 @@ public void testUpdatePriorityOnPendingAppAndKillAttempt() throws Exception { CSQueue defaultQueue = (LeafQueue) cs.getQueue("default"); // Update priority and kill application with no resource - RMApp app1 = rm.submitApp(1024, Priority.newInstance(appPriority)); + MockRMAppSubmissionData data3 = MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm) + .withPriority(Priority.newInstance(appPriority)) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data3); Collection appsPending = ((LeafQueue) defaultQueue).getPendingApplications(); Collection activeApps = @@ -756,11 +806,20 @@ public void testUpdatePriorityOnPendingAppAndKillAttempt() throws Exception { MockNM nm1 = new MockNM("127.0.0.1:1234", 8096, rm.getResourceTrackerService()); nm1.registerNode(); - RMApp app2 = rm.submitApp(1024, Priority.newInstance(appPriority)); + MockRMAppSubmissionData data2 = MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm) + .withPriority(Priority.newInstance(appPriority)) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm, data2); Assert.assertEquals("Pending apps should be 0", 0, appsPending.size()); Assert.assertEquals("Active apps should be 1", 1, activeApps.size()); - RMApp app3 = rm.submitApp(1024, Priority.newInstance(appPriority)); - RMApp app4 = rm.submitApp(1024, Priority.newInstance(appPriority)); + MockRMAppSubmissionData data1 = MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm) + .withPriority(Priority.newInstance(appPriority)) + .build(); + RMApp app3 = MockRMAppSubmitter.submit(rm, data1); + MockRMAppSubmissionData data = MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm) + .withPriority(Priority.newInstance(appPriority)) + .build(); + RMApp app4 = MockRMAppSubmitter.submit(rm, data); Assert.assertEquals("Pending apps should be 2", 2, appsPending.size()); Assert.assertEquals("Active apps should be 1", 1, activeApps.size()); // kill app3, pending apps should reduce to 1 diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index c090866ca4f..103411fdd91 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -98,6 +98,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockNodes; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.NodeManager; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; @@ -1274,7 +1276,7 @@ public void testResourceOverCommit() throws Exception { rm.start(); MockNM nm1 = rm.registerNode("127.0.0.1:1234", 4 * GB); - RMApp app1 = rm.submitApp(2048); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(2048, rm); // kick the scheduling, 2 GB given to AM1, remaining 2GB on nm1 nm1.nodeHeartbeat(true); RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); @@ -1490,7 +1492,13 @@ public void testAllocateDoesNotBlockOnSchedulerLock() throws Exception { Map acls = new HashMap(2); acls.put(ApplicationAccessType.VIEW_APP, "*"); - RMApp app = rm.submitApp(1024, "appname", "appuser", acls); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm) + .withName("appname") + .withUser("appuser") + .withAcls(acls) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); nm1.nodeHeartbeat(true); @@ -1636,7 +1644,7 @@ public void testPreemptionInfo() throws Exception { nm1.registerNode(); // create app and launch the AM - RMApp app0 = rm1.submitApp(CONTAINER_MEMORY); + RMApp app0 = MockRMAppSubmitter.submitWithMemory(CONTAINER_MEMORY, rm1); MockAM am0 = MockRM.launchAM(app0, rm1, nm1); am0.registerAppAttempt(); @@ -1702,7 +1710,7 @@ public void testRecoverRequestAfterPreemption() throws Exception { MockRM rm1 = new MockRM(conf); rm1.start(); MockNM nm1 = rm1.registerNode("127.0.0.1:1234", 8000); - RMApp app1 = rm1.submitApp(1024); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(1024, rm1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler(); @@ -1781,7 +1789,15 @@ public void testMoveAppBasic() throws Exception { (AbstractYarnScheduler) rm.getResourceScheduler(); // submit an app - RMApp app = rm.submitApp(GB, "test-move-1", "user_0", null, "a1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("test-move-1") + .withUser("user_0") + .withAcls(null) + .withQueue("a1") + .withUnmanaged(false) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); ApplicationAttemptId appAttemptId = rm.getApplicationReport(app.getApplicationId()) .getCurrentApplicationAttemptId(); @@ -1843,7 +1859,15 @@ public void testMoveAppSameParent() throws Exception { (AbstractYarnScheduler) rm.getResourceScheduler(); // submit an app - RMApp app = rm.submitApp(GB, "test-move-1", "user_0", null, "a1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("test-move-1") + .withUser("user_0") + .withAcls(null) + .withQueue("a1") + .withUnmanaged(false) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); ApplicationAttemptId appAttemptId = rm.getApplicationReport(app.getApplicationId()) .getCurrentApplicationAttemptId(); @@ -2359,7 +2383,15 @@ public void testMoveAllApps() throws Exception { (AbstractYarnScheduler) rm.getResourceScheduler(); // submit an app - RMApp app = rm.submitApp(GB, "test-move-1", "user_0", null, "a1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("test-move-1") + .withUser("user_0") + .withAcls(null) + .withQueue("a1") + .withUnmanaged(false) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); ApplicationAttemptId appAttemptId = rm.getApplicationReport(app.getApplicationId()) .getCurrentApplicationAttemptId(); @@ -2421,7 +2453,15 @@ public void testMoveAllAppsInvalidDestination() throws Exception { YarnScheduler scheduler = rm.getResourceScheduler(); // submit an app - RMApp app = rm.submitApp(GB, "test-move-1", "user_0", null, "a1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("test-move-1") + .withUser("user_0") + .withAcls(null) + .withQueue("a1") + .withUnmanaged(false) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); ApplicationAttemptId appAttemptId = rm.getApplicationReport(app.getApplicationId()) .getCurrentApplicationAttemptId(); @@ -2479,7 +2519,15 @@ public void testMoveAllAppsInvalidSource() throws Exception { YarnScheduler scheduler = rm.getResourceScheduler(); // submit an app - RMApp app = rm.submitApp(GB, "test-move-1", "user_0", null, "a1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("test-move-1") + .withUser("user_0") + .withAcls(null) + .withQueue("a1") + .withUnmanaged(false) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); ApplicationAttemptId appAttemptId = rm.getApplicationReport(app.getApplicationId()) .getCurrentApplicationAttemptId(); @@ -2737,7 +2785,15 @@ public void testKillAllAppsInQueue() throws Exception { (AbstractYarnScheduler) rm.getResourceScheduler(); // submit an app - RMApp app = rm.submitApp(GB, "test-move-1", "user_0", null, "a1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("test-move-1") + .withUser("user_0") + .withAcls(null) + .withQueue("a1") + .withUnmanaged(false) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); ApplicationAttemptId appAttemptId = rm.getApplicationReport(app.getApplicationId()) .getCurrentApplicationAttemptId(); @@ -2782,7 +2838,15 @@ public void testKillAllAppsInvalidSource() throws Exception { YarnScheduler scheduler = rm.getResourceScheduler(); // submit an app - RMApp app = rm.submitApp(GB, "test-move-1", "user_0", null, "a1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("test-move-1") + .withUser("user_0") + .withAcls(null) + .withQueue("a1") + .withUnmanaged(false) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); ApplicationAttemptId appAttemptId = rm.getApplicationReport(app.getApplicationId()) .getCurrentApplicationAttemptId(); @@ -2843,7 +2907,7 @@ public void testAppReservationWithDominantResourceCalculator() throws Exception MockNM nm2 = rm.registerNode("127.0.0.1:1235", 10 * GB, 4); rm.registerNode("127.0.0.1:1236", 10 * GB, 4); - RMApp app1 = rm.submitApp(1024); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(1024, rm); // kick the scheduling nm1.nodeHeartbeat(true); RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); @@ -3244,7 +3308,15 @@ public void testSchedulerKeyGarbageCollection() throws Exception { nm3.registerNode(); nm4.registerNode(); - RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); ApplicationAttemptId attemptId = app1.getCurrentAppAttempt().getAppAttemptId(); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm2); @@ -3395,7 +3467,15 @@ public void testHierarchyQueuesCurrentLimits() throws Exception { new MockNM("127.0.0.1:1234", 100 * GB, rm1.getResourceTrackerService()); nm1.registerNode(); - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "b1"); + MockRMAppSubmissionData data2 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data2); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); waitContainerAllocated(am1, 1 * GB, 1, 2, rm1, nm1); @@ -3405,7 +3485,15 @@ public void testHierarchyQueuesCurrentLimits() throws Exception { Assert.assertEquals(69 * GB, am1.doHeartbeat().getAvailableResources().getMemorySize()); - RMApp app2 = rm1.submitApp(1 * GB, "app", "user", null, "b2"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b2") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data1); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1); // Allocate 5 containers, each one is 8 GB in am2 (40 GB in total) @@ -3423,7 +3511,15 @@ public void testHierarchyQueuesCurrentLimits() throws Exception { // Now we submit app3 to a1 (in higher level hierarchy), to see if headroom // of app1 (in queue b1) updated correctly - RMApp app3 = rm1.submitApp(1 * GB, "app", "user", null, "a1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withUnmanaged(false) + .build(); + RMApp app3 = MockRMAppSubmitter.submit(rm1, data); MockAM am3 = MockRM.launchAndRegisterAM(app3, rm1, nm1); // Allocate 3 containers, each one is 8 GB in am3 (24 GB in total) @@ -3476,12 +3572,28 @@ public void testParentQueueMaxCapsAreRespected() throws Exception { nm1.registerNode(); // Launch app1 in a1, resource usage is 1GB (am) + 4GB * 2 = 9GB - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "a1"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); waitContainerAllocated(am1, 4 * GB, 2, 2, rm1, nm1); // Try to launch app2 in a2, asked 2GB, should success - RMApp app2 = rm1.submitApp(2 * GB, "app", "user", null, "a2"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(2 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a2") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1); try { // Try to allocate a container, a's usage=11G/max=12 @@ -3524,11 +3636,27 @@ protected RMNodeLabelsManager createNodeLabelManager() { nm2.registerNode(); // Launch app1 in queue=a1 - RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "a1"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm2); // Launch app2 in queue=b1 - RMApp app2 = rm.submitApp(8 * GB, "app", "user", null, "b1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(8 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm2); // am1 asks for 8 * 1GB container for no label @@ -3644,7 +3772,15 @@ public void testAMUsedResource() throws Exception { amMemory < minAllocMb); Resource minAllocResource = Resource.newInstance(minAllocMb, 1); String queueName = "a1"; - RMApp rmApp = rm.submitApp(amMemory, "app-1", "user_0", null, queueName); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(amMemory, rm) + .withName("app-1") + .withUser("user_0") + .withAcls(null) + .withQueue(queueName) + .withUnmanaged(false) + .build(); + RMApp rmApp = MockRMAppSubmitter.submit(rm, data); assertEquals("RMApp does not containes minimum allocation", minAllocResource, rmApp.getAMResourceRequests().get(0).getCapability()); @@ -3913,7 +4049,15 @@ protected RMNodeLabelsManager createNodeLabelManager() { nm1.registerNode(); // Launch app1 in queue=a1 - RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "a1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); // Allocate two more containers @@ -4020,9 +4164,23 @@ public Boolean get() { } }, 100, 60 * 1000); - rm.submitApp(amResource1, "app-1", userName, null, queueName); - - rm.submitApp(amResource2, "app-2", userName, null, queueName); + MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(amResource1, rm) + .withResource(amResource1) + .withName("app-1") + .withUser(userName) + .withAcls(null) + .withQueue(queueName) + .build()); + + MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(amResource2, rm) + .withResource(amResource2) + .withName("app-2") + .withUser(userName) + .withAcls(null) + .withQueue(queueName) + .build()); // When AM limit is exceeded, 1 applications will be activated.Rest all // applications will be in pending @@ -4194,7 +4352,7 @@ public void testSchedulingOnRemovedNode() throws Exception { MockRM rm = new MockRM(conf); rm.start(); - RMApp app = rm.submitApp(100); + RMApp app = MockRMAppSubmitter.submitWithMemory(100, rm); rm.drainEvents(); MockNM nm1 = rm.registerNode("127.0.0.1:1234", 10240, 10); @@ -4476,7 +4634,15 @@ protected RMNodeLabelsManager createNodeLabelManager() { nm1.registerNode(); // Launch app1 in queue=a1 - RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "a"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); // Got one offswitch request and offswitch allocation MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); @@ -4810,7 +4976,15 @@ public void testQueueMappingWithCurrentUserQueueMappingForaGroup() throws CapacityScheduler cs = ((CapacityScheduler) rm.getResourceScheduler()); cs.start(); - RMApp app = rm.submitApp(GB, "appname", "a1", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("appname") + .withUser("a1") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); List appsInA1 = cs.getAppsInQueue("a1"); assertEquals(1, appsInA1.size()); } @@ -4914,7 +5088,12 @@ public void testClearRequestsBeforeApplyTheProposal() final MockNM nm = rm.registerNode("h1:1234", 200 * GB); // submit app - final RMApp app = rm.submitApp(200, "app", "user"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm) + .withName("app") + .withUser("user") + .build(); + final RMApp app = MockRMAppSubmitter.submit(rm, data); MockRM.launchAndRegisterAM(app, rm, nm); // spy capacity scheduler to handle CapacityScheduler#apply @@ -4978,7 +5157,12 @@ public void testContainerAllocationLocalitySkipped() throws Exception { // submit app and request resource // container2 is larger than nm1 total resource, will trigger locality skip - final RMApp app = rm.submitApp(1 * GB, "app", "user"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .build(); + final RMApp app = MockRMAppSubmitter.submit(rm, data); final MockAM am = MockRM.launchAndRegisterAM(app, rm, nm1); am.addRequests(new String[] {"*"}, 5 * GB, 1, 1, 2); am.schedule(); @@ -5039,19 +5223,51 @@ public void testMoveAppWithActiveUsersWithOnlyPendingApps() throws Exception { MockNM nm1 = rm.registerNode("h1:1234", 16 * GB); // submit an app - RMApp app = rm.submitApp(GB, "test-move-1", "u1", null, "a1"); + MockRMAppSubmissionData data3 = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("test-move-1") + .withUser("u1") + .withAcls(null) + .withQueue("a1") + .withUnmanaged(false) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data3); MockAM am1 = MockRM.launchAndRegisterAM(app, rm, nm1); ApplicationAttemptId appAttemptId = rm.getApplicationReport(app.getApplicationId()) .getCurrentApplicationAttemptId(); - RMApp app2 = rm.submitApp(1 * GB, "app", "u2", null, "a1"); + MockRMAppSubmissionData data2 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("u2") + .withAcls(null) + .withQueue("a1") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm, data2); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm1); - RMApp app3 = rm.submitApp(1 * GB, "app", "u3", null, "a1"); - - RMApp app4 = rm.submitApp(1 * GB, "app", "u4", null, "a1"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("u3") + .withAcls(null) + .withQueue("a1") + .withUnmanaged(false) + .build(); + RMApp app3 = MockRMAppSubmitter.submit(rm, data1); + + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("u4") + .withAcls(null) + .withQueue("a1") + .withUnmanaged(false) + .build(); + RMApp app4 = MockRMAppSubmitter.submit(rm, data); // Each application asks 50 * 1GB containers am1.allocate("*", 1 * GB, 50, null); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java index 67c504d3df1..1de0a43904c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java @@ -33,11 +33,12 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; -import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEvent; @@ -146,8 +147,18 @@ public RMNodeLabelsManager createNodeLabelManager() { int totalAsked = 3 * GB; // 3 AMs for (int i = 0; i < 3; i++) { - RMApp rmApp = rm.submitApp(1024, "app", "user", null, false, - Character.toString((char) (i % 34 + 97)), 1, null, null, false); + RMApp rmApp = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withUnmanaged(false) + .withQueue(Character.toString((char) (i % 34 + 97))) + .withMaxAppAttempts(1) + .withCredentials(null) + .withAppType(null) + .withWaitForAccepted(false) + .build()); MockAM am = MockRM.launchAMWhenAsyncSchedulingEnabled(rmApp, rm); am.registerAppAttempt(); ams.add(am); @@ -220,8 +231,20 @@ public void testCommitProposalForFailedAppAttempt() SchedulerNode sn2 = scheduler.getSchedulerNode(nm2.getNodeId()); // launch app - RMApp app = rm.submitApp(200, "app", "user", null, false, "default", - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS, null, null, true, true); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS) + .withCredentials(null) + .withAppType(null) + .withWaitForAccepted(true) + .withKeepContainers(true) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); MockAM am = MockRM.launchAndRegisterAM(app, rm, nm1); FiCaSchedulerApp schedulerApp = scheduler.getApplicationAttempt(am.getApplicationAttemptId()); @@ -313,10 +336,26 @@ public void testCommitOutdatedReservedProposal() throws Exception { ((CapacityScheduler) scheduler).getSchedulerNode(nm2.getNodeId()); // submit app1, am1 is running on nm1 - RMApp app = rm.submitApp(200, "app", "user", null, "default"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data1); final MockAM am = MockRM.launchAndRegisterAM(app, rm, nm1); // submit app2, am2 is running on nm1 - RMApp app2 = rm.submitApp(200, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm, data); final MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm1); // allocate and launch 2 containers for app1 @@ -450,8 +489,20 @@ public void testNodeResourceOverAllocated() SchedulerNode sn1 = scheduler.getSchedulerNode(nm1.getNodeId()); // launch app - RMApp app = rm.submitApp(200, "app", "user", null, false, "default", - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS, null, null, true, true); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS) + .withCredentials(null) + .withAppType(null) + .withWaitForAccepted(true) + .withKeepContainers(true) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); MockAM am = MockRM.launchAndRegisterAM(app, rm, nm1); FiCaSchedulerApp schedulerApp = scheduler.getApplicationAttempt(am.getApplicationAttemptId()); @@ -532,8 +583,18 @@ public RMNodeLabelsManager createNodeLabelManager() { keepNMHeartbeat(nms, heartbeatInterval); for (int i = 0; i < 3; i++) { - RMApp rmApp = rm.submitApp(1024, "app", "user", null, false, - Character.toString((char) (i % 34 + 97)), 1, null, null, false); + RMApp rmApp = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withUnmanaged(false) + .withQueue(Character.toString((char) (i % 34 + 97))) + .withMaxAppAttempts(1) + .withCredentials(null) + .withAppType(null) + .withWaitForAccepted(false) + .build()); MockAM am = MockRM.launchAMWhenAsyncSchedulingEnabled(rmApp, rm); am.registerAppAttempt(); ams.add(am); @@ -632,8 +693,20 @@ public void testCommitDuplicatedAllocateFromReservedProposals() SchedulerNode sn1 = cs.getSchedulerNode(nm1.getNodeId()); // launch app - RMApp app = rm.submitApp(1 * GB, "app", "user", null, false, "default", - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS, null, null, true, true); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS) + .withCredentials(null) + .withAppType(null) + .withWaitForAccepted(true) + .withKeepContainers(true) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); MockAM am = MockRM.launchAndRegisterAM(app, rm, nm1); FiCaSchedulerApp schedulerApp = cs.getApplicationAttempt(am.getApplicationAttemptId()); @@ -720,7 +793,15 @@ public void testReleaseOutdatedReservedContainer() throws Exception { SchedulerNode sn3 = cs.getSchedulerNode(nm3.getNodeId()); // launch another app to queue, AM container should be launched in nm1 - RMApp app1 = rm1.submitApp(4 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(4 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); Resource allocateResource = Resources.createResource(5 * GB); am1.allocate("*", (int) allocateResource.getMemorySize(), 3, 0, @@ -771,13 +852,37 @@ public void testCommitProposalsForUnusableNode() throws Exception { SchedulerNode sn1 = cs.getSchedulerNode(nm1.getNodeId()); // launch app1-am on nm1 - RMApp app1 = rm.submitApp(1 * GB, "app1", "user", null, false, "default", - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS, null, null, true, true); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app1") + .withUser("user") + .withAcls(null) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS) + .withCredentials(null) + .withAppType(null) + .withWaitForAccepted(true) + .withKeepContainers(true) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); // launch app2-am on nm2 - RMApp app2 = rm.submitApp(1 * GB, "app2", "user", null, false, "default", - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS, null, null, true, true); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app2") + .withUser("user") + .withAcls(null) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS) + .withCredentials(null) + .withAppType(null) + .withWaitForAccepted(true) + .withKeepContainers(true) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm2); // app2 asks 1 * 8G container @@ -865,8 +970,20 @@ public void testReturnNullWhenGetSchedulerContainer() throws Exception { SchedulerNode sn2 = cs.getSchedulerNode(nm2.getNodeId()); // launch app1-am on nm1 - RMApp app1 = rm.submitApp(1 * GB, "app1", "user", null, false, "default", - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS, null, null, true, true); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app1") + .withUser("user") + .withAcls(null) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS) + .withCredentials(null) + .withAppType(null) + .withWaitForAccepted(true) + .withKeepContainers(true) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); // app2 asks 1 * 1G container diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java index f313d70191b..ee918a165d4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java @@ -23,7 +23,6 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.CommonConfigurationKeys; -import org.apache.hadoop.security.GroupMappingServiceProvider; import org.apache.hadoop.security.ShellBasedUnixGroupsMapping; import org.apache.hadoop.security.TestGroupsCaching; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; @@ -38,6 +37,8 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels .NullRMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; @@ -63,8 +64,6 @@ .AppAttemptAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event .SchedulerEvent; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair - .SimpleGroupsMapping; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.util.YarnVersionInfo; @@ -468,8 +467,15 @@ protected ApplicationId submitApp(MockRM rm, CSQueue parentQueue, CapacityScheduler capacityScheduler = (CapacityScheduler) rm.getResourceScheduler(); // submit an app - RMApp rmApp = rm.submitApp(GB, "test-auto-queue-activation", user, null, - leafQueueName); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("test-auto-queue-activation") + .withUser(user) + .withAcls(null) + .withQueue(leafQueueName) + .withUnmanaged(false) + .build(); + RMApp rmApp = MockRMAppSubmitter.submit(rm, data); // check preconditions List appsInParentQueue = @@ -549,9 +555,15 @@ protected ApplicationAttemptId submitApp(CapacityScheduler newCS, String user, protected RMApp submitApp(String user, String queue, String nodeLabel) throws Exception { - RMApp app = mockRM.submitApp(GB, - "test-auto-queue-creation" + RandomUtils.nextInt(0, 100), user, null, - queue, nodeLabel); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, mockRM) + .withName("test-auto-queue-creation" + RandomUtils.nextInt(0, 100)) + .withUser(user) + .withAcls(null) + .withQueue(queue) + .withAmLabel(nodeLabel) + .build(); + RMApp app = MockRMAppSubmitter.submit(mockRM, data); Assert.assertEquals(app.getAmNodeLabelExpression(), nodeLabel); // check preconditions List appsInC = cs.getAppsInQueue(PARENT_QUEUE); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java index 0a530dd3c9b..64cec46c8b7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java @@ -31,6 +31,8 @@ import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.resourcemanager.MockNodes; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; import org.apache.hadoop.yarn.server.resourcemanager.placement @@ -70,7 +72,6 @@ import org.junit.Test; import java.io.IOException; -import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -170,11 +171,25 @@ public void testReinitializeStoppedAutoCreatedLeafQueue() throws Exception { // submit an app - RMApp app1 = mockRM.submitApp(GB, "test-auto-queue-creation-1", USER0, - null, USER0); - - RMApp app2 = mockRM.submitApp(GB, "test-auto-queue-creation-2", USER1, - null, USER1); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, mockRM) + .withName("test-auto-queue-creation-1") + .withUser(USER0) + .withAcls(null) + .withQueue(USER0) + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(mockRM, data1); + + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, mockRM) + .withName("test-auto-queue-creation-2") + .withUser(USER1) + .withAcls(null) + .withQueue(USER1) + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(mockRM, data); // check preconditions List appsInC = cs.getAppsInQueue(PARENT_QUEUE); assertEquals(2, appsInC.size()); @@ -352,8 +367,14 @@ public void testAutoCreateLeafQueueFailsWithNoQueueMapping() // submit an app under a different queue name which does not exist // and queue mapping does not exist for this user - RMApp app = mockRM.submitApp(GB, "app", INVALID_USER, null, INVALID_USER, - false); + RMApp app = MockRMAppSubmitter.submit(mockRM, + MockRMAppSubmissionData.Builder.createWithDefaults(GB, mockRM) + .withName("app") + .withUser(INVALID_USER) + .withAcls(null) + .withQueue(INVALID_USER) + .withWaitForAccepted(false) + .build()); mockRM.drainEvents(); mockRM.waitForState(app.getApplicationId(), RMAppState.FAILED); assertEquals(RMAppState.FAILED, app.getState()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java index 9aba30c2e88..ca879ef5610 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java @@ -29,6 +29,8 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationConstants; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; @@ -168,7 +170,15 @@ public void testRemoveQueue() throws Exception { a1.setEntitlement(new QueueEntitlement(A1_CAPACITY / 100, 1f)); // submit an app - RMApp app = rm.submitApp(GB, "test-move-1", "user_0", null, "a1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("test-move-1") + .withUser("user_0") + .withAcls(null) + .withQueue("a1") + .withUnmanaged(false) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); // check preconditions List appsInA1 = cs.getAppsInQueue("a1"); assertEquals(1, appsInA1.size()); @@ -203,7 +213,15 @@ public void testMoveAppToPlanQueue() throws Exception { CapacityScheduler scheduler = (CapacityScheduler) rm.getResourceScheduler(); // submit an app - RMApp app = rm.submitApp(GB, "test-move-1", "user_0", null, "b1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("test-move-1") + .withUser("user_0") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); ApplicationAttemptId appAttemptId = rm.getApplicationReport(app.getApplicationId()) .getCurrentApplicationAttemptId(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerLazyPreemption.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerLazyPreemption.java index a4c7d61fff7..9af622f6496 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerLazyPreemption.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerLazyPreemption.java @@ -25,7 +25,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; -import org.apache.hadoop.yarn.server.resourcemanager.monitor.SchedulingEditPolicy; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.monitor.SchedulingMonitor; import org.apache.hadoop.yarn.server.resourcemanager.monitor.SchedulingMonitorManager; import org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy; @@ -96,7 +97,15 @@ public void testSimplePreemption() throws Exception { RMNode rmNode2 = rm1.getRMContext().getRMNodes().get(nm2.getNodeId()); // launch an app to queue, AM container should be launched in nm1 - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "a"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); am1.allocate("*", 1 * GB, 7, new ArrayList()); @@ -113,7 +122,15 @@ public void testSimplePreemption() throws Exception { Assert.assertEquals(7, schedulerApp1.getLiveContainers().size()); // Submit app2 to queue-c and asks for a 1G container for AM - RMApp app2 = rm1.submitApp(1 * GB, "app", "user", null, "c"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm2); // NM1/NM2 has available resource = 0G @@ -179,7 +196,15 @@ public void testPreemptionConsidersNodeLocalityDelay() RMNode rmNode2 = rm1.getRMContext().getRMNodes().get(nm2.getNodeId()); // launch an app to queue, AM container should be launched in nm1 - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "a"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); am1.allocate("*", 1 * GB, 6, new ArrayList()); @@ -196,7 +221,15 @@ public void testPreemptionConsidersNodeLocalityDelay() Assert.assertEquals(7, schedulerApp1.getLiveContainers().size()); // Submit app2 to queue-c and asks for a 1G container for AM - RMApp app2 = rm1.submitApp(1 * GB, "app", "user", null, "c"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm2); // NM1/NM2 has available resource = 0G @@ -272,7 +305,15 @@ public void testPreemptionConsidersHardNodeLocality() RMNode rmNode2 = rm1.getRMContext().getRMNodes().get(nm2.getNodeId()); // launch an app to queue, AM container should be launched in nm1 - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "a"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); am1.allocate("*", 1 * GB, 6, new ArrayList()); @@ -291,7 +332,15 @@ public void testPreemptionConsidersHardNodeLocality() Assert.assertEquals(7, schedulerApp1.getLiveContainers().size()); // Submit app2 to queue-c and asks for a 1G container for AM - RMApp app2 = rm1.submitApp(1 * GB, "app", "user", null, "c"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm2); // NM1/NM2 has available resource = 0G @@ -376,7 +425,15 @@ public void testPreemptionPolicyShouldRespectAlreadyMarkedKillableContainers() RMNode rmNode1 = rm1.getRMContext().getRMNodes().get(nm1.getNodeId()); // launch an app to queue, AM container should be launched in nm1 - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "a"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); am1.allocate("*", 1 * GB, 6, new ArrayList()); @@ -392,7 +449,15 @@ public void testPreemptionPolicyShouldRespectAlreadyMarkedKillableContainers() Assert.assertEquals(7, schedulerApp1.getLiveContainers().size()); // Submit app2 to queue-c and asks for a 1G container for AM - RMApp app2 = rm1.submitApp(1 * GB, "app", "user", null, "c"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1); // NM1 has available resource = 0G @@ -479,7 +544,15 @@ public void testPreemptionPolicyCleanupKillableContainersWhenNoPreemptionNeeded( RMNode rmNode1 = rm1.getRMContext().getRMNodes().get(nm1.getNodeId()); // launch an app to queue, AM container should be launched in nm1 - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "a"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); am1.allocate("*", 1 * GB, 6, new ArrayList()); @@ -495,7 +568,15 @@ public void testPreemptionPolicyCleanupKillableContainersWhenNoPreemptionNeeded( Assert.assertEquals(7, schedulerApp1.getLiveContainers().size()); // Submit app2 to queue-c and asks for a 1G container for AM - RMApp app2 = rm1.submitApp(1 * GB, "app", "user", null, "c"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1); // NM1 has available resource = 0G @@ -570,7 +651,15 @@ public void testPreemptionConsidersUserLimit() RMNode rmNode2 = rm1.getRMContext().getRMNodes().get(nm2.getNodeId()); // launch an app to queue, AM container should be launched in nm1 - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "a"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); am1.allocate("*", 1 * GB, 6, new ArrayList()); @@ -587,7 +676,15 @@ public void testPreemptionConsidersUserLimit() Assert.assertEquals(7, schedulerApp1.getLiveContainers().size()); // Submit app2 to queue-c and asks for a 1G container for AM - RMApp app2 = rm1.submitApp(1 * GB, "app", "user", null, "c"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm2); // NM1/NM2 has available resource = 0G diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerMultiNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerMultiNodes.java index c90af94a169..62b17cec4b6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerMultiNodes.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerMultiNodes.java @@ -30,6 +30,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode; @@ -117,7 +119,15 @@ public void testMultiNodeSorterForSchedulingWithOrdering() throws Exception { .getNodesPerPartition(""); Assert.assertEquals(4, nodes.size()); - RMApp app1 = rm.submitApp(2048, "app-1", "user1", null, "default"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(2048, rm) + .withName("app-1") + .withUser("user1") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); SchedulerNodeReport reportNm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); @@ -131,7 +141,15 @@ public void testMultiNodeSorterForSchedulingWithOrdering() throws Exception { // Hence forcefully recompute nodes. sorter.reSortClusterNodes(); - RMApp app2 = rm.submitApp(1024, "app-2", "user2", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm) + .withName("app-2") + .withUser("user2") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm2); SchedulerNodeReport reportNm2 = rm.getResourceScheduler().getNodeReport(nm2.getNodeId()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java index e239191e5e3..4841626ec7c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java @@ -38,6 +38,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; @@ -220,7 +222,15 @@ public RMNodeLabelsManager createNodeLabelManager() { ContainerId containerId; // launch an app to queue a1 (label = x), and check all container will // be allocated in h1 - RMApp app1 = rm.submitApp(GB, "app", "user", null, "a"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm3); ApplicationResourceUsageReport appResourceUsageReport = rm.getResourceScheduler().getAppResourceUsageReport( @@ -285,7 +295,15 @@ public RMNodeLabelsManager createNodeLabelManager() { // launch an app to queue a1 (label = x), and check all container will // be allocated in h1 - RMApp app1 = rm.submitApp(GB, "app", "user", null, "a"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm3); // request a container. @@ -446,7 +464,15 @@ public RMNodeLabelsManager createNodeLabelManager() { MockNM nm4 = rm.registerNode("h4:1234", 4096 * 2); // launch an app to queue a1 (label = x), and check all container will // be allocated in h1 - RMApp app1 = rm.submitApp(GB, "app", "user", null, "a1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm3); am1.allocate("*", GB, 1, new ArrayList(), "x"); ContainerId container1 = @@ -524,7 +550,15 @@ public RMNodeLabelsManager createNodeLabelManager() { MockNM nm2 = rm.registerNode("h2:1234", 80000); // app1 - RMApp app1 = rm.submitApp(GB, "app", "u1", null, "a"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("app") + .withUser("u1") + .withAcls(null) + .withQueue("a") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm2); // c2 on n1, c3 on n2 @@ -540,7 +574,15 @@ public RMNodeLabelsManager createNodeLabelManager() { RMContainerState.ALLOCATED)); // app2 - RMApp app2 = rm.submitApp(GB, "app", "u2", null, "a"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("app") + .withUser("u2") + .withAcls(null) + .withQueue("a") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm2); // c2/c3 on n1 @@ -664,7 +706,15 @@ public RMNodeLabelsManager createNodeLabelManager() { // launch an app to queue a1 (label = x), and check all container will // be allocated in h1 - RMApp app1 = rm.submitApp(GB, "app", "user", null, "a", "x"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a") + .withAmLabel("x") + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); // request a container. @@ -760,7 +810,15 @@ public RMNodeLabelsManager createNodeLabelManager() { rm.registerNode("h9:1234", 2048); // Submit app with AM container launched on default partition i.e. h1. - RMApp app = rm.submitApp(GB, "app", "user", null, "a"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a") + .withUnmanaged(false) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); MockRM.launchAndRegisterAM(app, rm, nm1); RMAppAttempt appAttempt = app.getCurrentAppAttempt(); // Add default node blacklist from default @@ -776,7 +834,15 @@ public RMNodeLabelsManager createNodeLabelManager() { Assert.assertEquals(2, blacklistUpdates.getBlacklistRemovals().size()); // Submission in label x - RMApp applabel = rm.submitApp(GB, "app", "user", null, "a", "x"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a") + .withAmLabel("x") + .build(); + RMApp applabel = MockRMAppSubmitter.submit(rm, data1); MockRM.launchAndRegisterAM(applabel, rm, nm2); RMAppAttempt appAttemptlabelx = applabel.getCurrentAppAttempt(); appAttemptlabelx.getAMBlacklistManager().addNode("h2"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSchedulingRequestUpdate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSchedulingRequestUpdate.java index ee7e013ccbd..32dcc8ad8c6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSchedulingRequestUpdate.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSchedulingRequestUpdate.java @@ -30,6 +30,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; @@ -75,11 +77,27 @@ protected RMNodeLabelsManager createNodeLabelManager() { nm2.registerNode(); // Launch app1 in queue=a1 - RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "a1"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm2); // Launch app2 in queue=b1 - RMApp app2 = rm.submitApp(8 * GB, "app", "user", null, "b1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(8 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm2); // am1 asks for 8 * 1GB container for no label am1.allocateIntraAppAntiAffinity( @@ -193,11 +211,27 @@ protected RMNodeLabelsManager createNodeLabelManager() { nm2.registerNode(); // Launch app1 in queue=a1 - RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "a1"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm2); // Launch app2 in queue=b1 - RMApp app2 = rm.submitApp(8 * GB, "app", "user", null, "b1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(8 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm2); // am1 asks for 8 * 1GB container for "x" am1.allocateIntraAppAntiAffinity("x", diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java index 800789af72a..952097146ac 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java @@ -30,6 +30,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.monitor.SchedulingMonitor; import org.apache.hadoop.yarn.server.resourcemanager.monitor.SchedulingMonitorManager; import org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy; @@ -121,7 +123,15 @@ protected RMNodeLabelsManager createNodeLabelManager() { RMNode rmNode2 = rm1.getRMContext().getRMNodes().get(nm2.getNodeId()); // launch an app to queue, AM container should be launched in nm1 - RMApp app1 = rm1.submitApp(1 * GB, "app", user1, null, queue1); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser(user1) + .withAcls(null) + .withQueue(queue1) + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); am1.allocate("*", 1 * GB, 32, new ArrayList()); @@ -144,7 +154,15 @@ protected RMNodeLabelsManager createNodeLabelManager() { // Submit app2 to queue-c and asks for a 1G container for AM - RMApp app2 = rm1.submitApp(1 * GB, "app", user2, null, queue2); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser(user2) + .withAcls(null) + .withQueue(queue2) + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1); // NM1/NM2 has available resource = 2G/4G @@ -222,7 +240,15 @@ public void testSurgicalPreemptionWithAvailableResource() RMNode rmNode2 = rm1.getRMContext().getRMNodes().get(nm2.getNodeId()); // launch an app to queue, AM container should be launched in nm1 - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "a"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); am1.allocate("*", 1 * GB, 38, new ArrayList()); @@ -245,7 +271,15 @@ public void testSurgicalPreemptionWithAvailableResource() // Submit app2 to queue-c and asks for a 4G container for AM - RMApp app2 = rm1.submitApp(4 * GB, "app", "user", null, "c"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(4 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data); FiCaSchedulerApp schedulerApp2 = cs.getApplicationAttempt( ApplicationAttemptId.newInstance(app2.getApplicationId(), 1)); @@ -326,7 +360,15 @@ public void testPriorityPreemptionWhenAllQueuesAreBelowGuaranteedCapacities() RMNode rmNode2 = rm1.getRMContext().getRMNodes().get(nm2.getNodeId()); // launch an app to queue, AM container should be launched in nm1 - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "b"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); am1.allocate("*", 1 * GB, 6, new ArrayList<>()); @@ -349,7 +391,15 @@ public void testPriorityPreemptionWhenAllQueuesAreBelowGuaranteedCapacities() am1.getApplicationAttemptId(), 3); // Submit app2 to queue-c and asks for a 1G container for AM - RMApp app2 = rm1.submitApp(18 * GB, "app", "user", null, "c"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(18 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data); FiCaSchedulerApp schedulerApp2 = cs.getApplicationAttempt( ApplicationAttemptId.newInstance(app2.getApplicationId(), 1)); @@ -435,7 +485,15 @@ public void testPriorityPreemptionRequiresMoveReservation() RMNode rmNode3 = rm1.getRMContext().getRMNodes().get(nm3.getNodeId()); // launch an app to queue, AM container should be launched in nm1 - RMApp app1 = rm1.submitApp(2 * GB, "app", "user", null, "b"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(2 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); am1.allocate("*", 2 * GB, 2, new ArrayList<>()); @@ -457,7 +515,15 @@ public void testPriorityPreemptionRequiresMoveReservation() am1.getApplicationAttemptId(), 2); // Submit app2 to queue-c and asks for a 2G container for AM, on n3 - RMApp app2 = rm1.submitApp(2 * GB, "app", "user", null, "c"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(2 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm3); FiCaSchedulerApp schedulerApp2 = cs.getApplicationAttempt( ApplicationAttemptId.newInstance(app2.getApplicationId(), 1)); @@ -560,7 +626,15 @@ public void testPriorityPreemptionOnlyTriggeredWhenDemandingQueueUnsatisfied() } // launch an app to queue, AM container should be launched in nm1 - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "b"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, mockNMs[0]); am1.allocate("*", 1 * GB, 8, new ArrayList<>()); @@ -581,7 +655,15 @@ public void testPriorityPreemptionOnlyTriggeredWhenDemandingQueueUnsatisfied() // Submit app2 to queue-c and asks for a 10G container for AM // Launch AM in NM9 - RMApp app2 = rm1.submitApp(10 * GB, "app", "user", null, "c"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(10 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, mockNMs[9]); FiCaSchedulerApp schedulerApp2 = cs.getApplicationAttempt( ApplicationAttemptId.newInstance(app2.getApplicationId(), 1)); @@ -704,7 +786,15 @@ protected RMNodeLabelsManager createNodeLabelManager() { } // launch an app to queue, AM container should be launched in nm1 - RMApp app1 = rm1.submitApp(1 * GB, "app", users[2], null, queues[2]); + MockRMAppSubmissionData data2 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser(users[2]) + .withAcls(null) + .withQueue(queues[2]) + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data2); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, mockNMs[4]); am1.allocate("*", 1 * GB, 4, new ArrayList<>()); @@ -724,7 +814,15 @@ protected RMNodeLabelsManager createNodeLabelManager() { } // Submit app2 to queue-a and asks for a 0.5G container for AM (on n0) - RMApp app2 = rm1.submitApp(512, "app", users[0], null, queues[0]); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(512, rm1) + .withName("app") + .withUser(users[0]) + .withAcls(null) + .withQueue(queues[0]) + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data1); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, mockNMs[0]); FiCaSchedulerApp schedulerApp2 = cs.getApplicationAttempt( ApplicationAttemptId.newInstance(app2.getApplicationId(), 1)); @@ -746,7 +844,15 @@ protected RMNodeLabelsManager createNodeLabelManager() { } // Submit app3 to queue-b and asks for a 0.5G container for AM (on n2) - RMApp app3 = rm1.submitApp(512, "app", users[1], null, queues[1]); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(512, rm1) + .withName("app") + .withUser(users[1]) + .withAcls(null) + .withQueue(queues[1]) + .withUnmanaged(false) + .build(); + RMApp app3 = MockRMAppSubmitter.submit(rm1, data); MockAM am3 = MockRM.launchAndRegisterAM(app3, rm1, mockNMs[2]); FiCaSchedulerApp schedulerApp3 = cs.getApplicationAttempt( ApplicationAttemptId.newInstance(app3.getApplicationId(), 1)); @@ -936,7 +1042,15 @@ protected RMNodeLabelsManager createNodeLabelManager() { } // launch an app to queue B, AM container launched in nm4 - RMApp app1 = rm1.submitApp(4096, "app", "user", null, "B"); + MockRMAppSubmissionData data2 = + MockRMAppSubmissionData.Builder.createWithDefaults(4096, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("B") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data2); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, mockNMs[4]); am1.allocate("*", 4096, NUM_NM-1, new ArrayList<>()); @@ -956,7 +1070,15 @@ protected RMNodeLabelsManager createNodeLabelManager() { } // Submit app2 to queue A and asks for a 750MB container for AM (on n0) - RMApp app2 = rm1.submitApp(1024, "app", "user", null, "A"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("A") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data1); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, mockNMs[0]); FiCaSchedulerApp schedulerApp2 = cs.getApplicationAttempt( ApplicationAttemptId.newInstance(app2.getApplicationId(), 1)); @@ -973,7 +1095,15 @@ protected RMNodeLabelsManager createNodeLabelManager() { rm1.killApp(app1.getApplicationId()); // Submit app3 to queue B and asks for a 5000MB container for AM (on n2) - RMApp app3 = rm1.submitApp(1024, "app", "user", null, "B"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("B") + .withUnmanaged(false) + .build(); + RMApp app3 = MockRMAppSubmitter.submit(rm1, data); MockAM am3 = MockRM.launchAndRegisterAM(app3, rm1, mockNMs[2]); FiCaSchedulerApp schedulerApp3 = cs.getApplicationAttempt( ApplicationAttemptId.newInstance(app3.getApplicationId(), 1)); @@ -1049,7 +1179,15 @@ public void testPreemptionForFragmentatedCluster() throws Exception { CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler(); // launch an app to queue, AM container should be launched in nm1 - RMApp app1 = rm1.submitApp(3 * GB, "app", "user", null, "a"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(3 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nms.get(0)); am1.allocate("*", 21 * GB, 4, new ArrayList()); @@ -1067,7 +1205,15 @@ public void testPreemptionForFragmentatedCluster() throws Exception { Assert.assertEquals(5, schedulerApp1.getLiveContainers().size()); // launch an app to queue, AM container should be launched in nm1 - RMApp app2 = rm1.submitApp(3 * GB, "app", "user", null, "b"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(3 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nms.get(2)); am2.allocate("*", 21 * GB, 4, new ArrayList()); @@ -1155,7 +1301,15 @@ public void testPreemptionToBalanceWithCustomTimeout() throws Exception { RMNode rmNode2 = rm1.getRMContext().getRMNodes().get(nm2.getNodeId()); // launch an app to queue, AM container should be launched in nm1 - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "a"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); am1.allocate("*", 1 * GB, 38, new ArrayList()); @@ -1178,7 +1332,15 @@ public void testPreemptionToBalanceWithCustomTimeout() throws Exception { // Submit app2 to queue-c and asks for a 4G container for AM - RMApp app2 = rm1.submitApp(4 * GB, "app", "user", null, "c"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(4 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data); FiCaSchedulerApp schedulerApp2 = cs.getApplicationAttempt( ApplicationAttemptId.newInstance(app2.getApplicationId(), 1)); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java index 93c35e4101e..80220cfb0cf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java @@ -44,6 +44,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; import org.apache.hadoop.yarn.server.resourcemanager.RMSecretManagerService; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager; @@ -115,7 +117,7 @@ public void testExcessReservationThanNodeManagerCapacity() throws Exception { } Assert.assertEquals(2, rm.getRMContext().getRMNodes().size()); // Submit an application - RMApp app1 = rm.submitApp(128); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(128, rm); // kick the scheduling nm1.nodeHeartbeat(true); @@ -167,7 +169,7 @@ public void testContainerTokenGeneratedOnPullRequest() throws Exception { MockRM rm1 = new MockRM(conf); rm1.start(); MockNM nm1 = rm1.registerNode("127.0.0.1:1234", 8000); - RMApp app1 = rm1.submitApp(200); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // request a container. am1.allocate("127.0.0.1", 1024, 1, new ArrayList()); @@ -196,7 +198,7 @@ public void testNormalContainerAllocationWhenDNSUnavailable() throws Exception{ MockRM rm1 = new MockRM(conf); rm1.start(); MockNM nm1 = rm1.registerNode("unknownhost:1234", 8000); - RMApp app1 = rm1.submitApp(200); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(200, rm1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // request a container. @@ -264,7 +266,10 @@ public void testLogAggregationContextPassedIntoContainerToken() private LogAggregationContext getLogAggregationContextFromContainerToken( MockRM rm1, MockNM nm1, LogAggregationContext logAggregationContext) throws Exception { - RMApp app2 = rm1.submitApp(200, logAggregationContext); + RMApp app2 = MockRMAppSubmitter.submit(rm1, + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withLogAggregationContext(logAggregationContext) + .build()); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1); nm1.nodeHeartbeat(true); // request a container. @@ -332,7 +337,7 @@ protected RMSecretManagerService createRMSecretManagerService() { RMApp app1; try { SecurityUtilTestHelper.setTokenServiceUseIp(true); - app1 = rm1.submitApp(200); + app1 = MockRMAppSubmitter.submitWithMemory(200, rm1); RMAppAttempt attempt = app1.getCurrentAppAttempt(); nm1.nodeHeartbeat(true); @@ -370,11 +375,27 @@ public void testExcessReservationWillBeUnreserved() throws Exception { MockNM nm2 = rm1.registerNode("h2:1234", 8 * GB); // launch an app to queue, AM container should be launched in nm1 - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // launch another app to queue, AM container should be launched in nm1 - RMApp app2 = rm1.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1); am1.allocate("*", 4 * GB, 1, new ArrayList()); @@ -453,11 +474,27 @@ public void testAllocationForReservedContainer() throws Exception { MockNM nm2 = rm1.registerNode("h2:1234", 8 * GB); // launch an app to queue, AM container should be launched in nm1 - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // launch another app to queue, AM container should be launched in nm1 - RMApp app2 = rm1.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1); am1.allocate("*", 4 * GB, 1, new ArrayList()); @@ -549,11 +586,27 @@ public void testReservedContainerMetricsOnDecommisionedNode() throws Exception { MockNM nm2 = rm1.registerNode("h2:1234", 8 * GB); // launch an app to queue, AM container should be launched in nm1 - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // launch another app to queue, AM container should be launched in nm1 - RMApp app2 = rm1.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1); am1.allocate("*", 4 * GB, 1, new ArrayList()); @@ -621,7 +674,15 @@ public void testAssignMultipleOffswitchContainers() throws Exception { MockNM nm1 = rm1.registerNode("h1:1234", 80 * GB); // launch an app to queue, AM container should be launched in nm1 - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); am1.allocate("*", 1 * GB, 5, new ArrayList()); @@ -670,12 +731,28 @@ public void testContinuousReservationLookingWhenUsedEqualsMax() throws Exception MockNM nm2 = rm1.registerNode("h2:1234", 90 * GB); // launch an app to queue A, AM container should be launched in nm1 - RMApp app1 = rm1.submitApp(2 * GB, "app", "user", null, "a"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(2 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // launch 2nd app to queue B, AM container should be launched in nm1 // Now usage of nm1 is 3G (2G + 1G) - RMApp app2 = rm1.submitApp(1 * GB, "app", "user", null, "b"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1); am1.allocate("*", 4 * GB, 2, null); @@ -722,11 +799,27 @@ public void testPendingResourcesConsideringUserLimit() throws Exception { MockNM nm2 = rm1.registerNode("h2:1234", 8 * GB); // launch an app to queue default, AM container should be launched in nm1 - RMApp app1 = rm1.submitApp(2 * GB, "app", "u1", null, "default"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(2 * GB, rm1) + .withName("app") + .withUser("u1") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // launch 2nd app to queue default, AM container should be launched in nm1 - RMApp app2 = rm1.submitApp(4 * GB, "app", "u2", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(4 * GB, rm1) + .withName("app") + .withUser("u2") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1); // am1 asks 1 * 3G container @@ -813,15 +906,39 @@ public void testQueuePriorityOrdering() throws Exception { MockNM nm1 = rm1.registerNode("h1:1234", 100 * GB); // launch an app to queue A, AM container should be launched in nm1 - RMApp app1 = rm1.submitApp(2 * GB, "app", "user", null, "a"); + MockRMAppSubmissionData data2 = + MockRMAppSubmissionData.Builder.createWithDefaults(2 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data2); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // launch an app to queue B, AM container should be launched in nm1 - RMApp app2 = rm1.submitApp(2 * GB, "app", "user", null, "b"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(2 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data1); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1); // launch an app to queue C, AM container should be launched in nm1 - RMApp app3 = rm1.submitApp(2 * GB, "app", "user", null, "c"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(2 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c") + .withUnmanaged(false) + .build(); + RMApp app3 = MockRMAppSubmitter.submit(rm1, data); MockAM am3 = MockRM.launchAndRegisterAM(app3, rm1, nm1); // Each application asks 10 * 5GB containers @@ -918,11 +1035,27 @@ public void testUserLimitAllocationMultipleContainers() throws Exception { MockNM nm1 = rm1.registerNode("h1:1234", 1000 * GB); // launch app from 1st user to queue C, AM container should be launched in nm1 - RMApp app1 = rm1.submitApp(2 * GB, "app", "user1", null, "c"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(2 * GB, rm1) + .withName("app") + .withUser("user1") + .withAcls(null) + .withQueue("c") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // launch app from 2nd user to queue C, AM container should be launched in nm1 - RMApp app2 = rm1.submitApp(2 * GB, "app", "user2", null, "c"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(2 * GB, rm1) + .withName("app") + .withUser("user2") + .withAcls(null) + .withQueue("c") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1); // Each application asks 1000 * 5GB containers @@ -965,15 +1098,47 @@ public void testActiveUsersWithOnlyPendingApps() throws Exception { rm1.start(); MockNM nm1 = rm1.registerNode("h1:1234", 8 * GB); - RMApp app1 = rm1.submitApp(1 * GB, "app", "u1", null, "default"); + MockRMAppSubmissionData data3 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("u1") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data3); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); - RMApp app2 = rm1.submitApp(1 * GB, "app", "u2", null, "default"); + MockRMAppSubmissionData data2 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("u2") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data2); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1); - RMApp app3 = rm1.submitApp(1 * GB, "app", "u3", null, "default"); - - RMApp app4 = rm1.submitApp(1 * GB, "app", "u4", null, "default"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("u3") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app3 = MockRMAppSubmitter.submit(rm1, data1); + + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("u4") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app4 = MockRMAppSubmitter.submit(rm1, data); // Each application asks 50 * 1GB containers am1.allocate("*", 1 * GB, 50, null); @@ -1030,11 +1195,27 @@ public void testUnreserveWhenClusterResourceHasEmptyResourceType() MockNM nm2 = rm1.registerNode("h2:1234", 8 * GB); // launch an app to queue "a", AM container should be launched on nm1 - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "a"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // launch another app to queue "b", AM container should be launched on nm1 - RMApp app2 = rm1.submitApp(1 * GB, "app", "user", null, "b"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data); MockRM.launchAndRegisterAM(app2, rm1, nm1); am1.allocate("*", 7 * GB, 2, new ArrayList()); @@ -1111,11 +1292,27 @@ public void testAllocationCannotBeBlockedWhenFormerQueueReachedItsLimit() MockNM nm2 = rm1.registerNode("h2:1234", 10 * GB); // launch an app to queue "c1", AM container should be launched on nm1 - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "c1"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // launch another app to queue "b", AM container should be launched on nm1 - RMApp app2 = rm1.submitApp(1 * GB, "app", "user", null, "b"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1); am1.allocate("*", 2 * GB, 1, new ArrayList()); @@ -1155,7 +1352,15 @@ public void testContainerRejectionWhenAskBeyondDynamicMax() // submit an app beyond queue max leads to failure. boolean submitFailed = false; MockNM nm1 = rm1.registerNode("h1:1234", 2 * GB, 1); - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "a"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); try { am1.allocate("*", 5 * GB, 1, null); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerResizing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerResizing.java index 307d5aea2a7..6e8339c9392 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerResizing.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerResizing.java @@ -42,6 +42,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; @@ -124,7 +126,15 @@ public RMNodeLabelsManager createNodeLabelManager() { MockNM nm1 = rm1.registerNode("h1:1234", 20 * GB); // app1 -> a1 - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); ContainerId containerId1 = @@ -182,7 +192,15 @@ protected Dispatcher createDispatcher() { MockNM nm1 = rm1.registerNode("h1:1234", 20 * GB); // app1 -> a1 - RMApp app1 = rm1.submitApp(3 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(3 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); FiCaSchedulerApp app = TestUtils.getFiCaSchedulerApp( rm1, app1.getApplicationId()); @@ -247,7 +265,15 @@ public RMNodeLabelsManager createNodeLabelManager() { MockNM nm2 = rm1.registerNode("h2:1234", 8 * GB); // app1 -> a1 - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); FiCaSchedulerApp app = TestUtils.getFiCaSchedulerApp( @@ -348,7 +374,15 @@ public RMNodeLabelsManager createNodeLabelManager() { MockNM nm1 = rm1.registerNode("h1:1234", 8 * GB); // app1 -> a1 - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); FiCaSchedulerApp app = TestUtils.getFiCaSchedulerApp( @@ -423,7 +457,15 @@ public RMNodeLabelsManager createNodeLabelManager() { MockNM nm2 = rm1.registerNode("h2:1234", 8 * GB); // app1 -> a1 - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); FiCaSchedulerApp app = TestUtils.getFiCaSchedulerApp( @@ -537,7 +579,15 @@ protected Dispatcher createDispatcher() { MockNM nm2 = rm1.registerNode("h2:1234", 8 * GB); // app1 -> a1 - RMApp app1 = rm1.submitApp(2 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(2 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); FiCaSchedulerApp app = TestUtils.getFiCaSchedulerApp( @@ -654,7 +704,15 @@ protected Dispatcher createDispatcher() { MockNM nm2 = rm1.registerNode("h2:1234", 8 * GB); // app1 -> a1 - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); FiCaSchedulerApp app = TestUtils.getFiCaSchedulerApp( @@ -758,7 +816,15 @@ public RMNodeLabelsManager createNodeLabelManager() { MockNM nm2 = rm1.registerNode("h2:1234", 8 * GB); // app1 -> a1 - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); FiCaSchedulerApp app = TestUtils.getFiCaSchedulerApp( @@ -882,7 +948,15 @@ public RMNodeLabelsManager createNodeLabelManager() { MockNM nm1 = rm1.registerNode("h1:1234", 10 * GB); // app1 -> a1 - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); FiCaSchedulerApp app = TestUtils.getFiCaSchedulerApp( @@ -963,7 +1037,15 @@ public ResourceScheduler createScheduler() { // register a node MockNM nm = rm.registerNode("h1:1234", 20 * GB); // submit an application -> app1 - RMApp app1 = rm.submitApp(3 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(3 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm); // making sure resource is allocated checkUsedResource(rm, "default", 3 * GB, null); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestIncreaseAllocationExpirer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestIncreaseAllocationExpirer.java index a800bef95ea..4ebfc0bbb50 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestIncreaseAllocationExpirer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestIncreaseAllocationExpirer.java @@ -33,6 +33,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; @@ -82,7 +84,15 @@ public void testContainerIsRemovedFromAllocationExpirer() rm1.start(); // Submit an application MockNM nm1 = rm1.registerNode("127.0.0.1:1234", 20 * GB); - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // Report AM container status RUNNING to remove it from expirer nm1.nodeHeartbeat( @@ -166,7 +176,15 @@ protected Dispatcher createDispatcher() { }; rm1.start(); MockNM nm1 = rm1.registerNode("127.0.0.1:1234", 20 * GB); - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); nm1.nodeHeartbeat( app1.getCurrentAppAttempt() @@ -242,7 +260,15 @@ public void testConsecutiveContainerIncreaseAllocationExpiration() rm1.start(); // Submit an application MockNM nm1 = rm1.registerNode("127.0.0.1:1234", 20 * GB); - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); nm1.nodeHeartbeat( app1.getCurrentAppAttempt() @@ -359,7 +385,15 @@ public void testDecreaseAfterIncreaseWithAllocationExpiration() rm1.start(); // Submit an application MockNM nm1 = rm1.registerNode("127.0.0.1:1234", 20 * GB); - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); nm1.nodeHeartbeat( app1.getCurrentAppAttempt() diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java index 9cfddd66f66..338d2bd4cff 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java @@ -37,6 +37,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; @@ -178,7 +180,15 @@ public RMNodeLabelsManager createNodeLabelManager() { // launch an app to queue a1 (label = x), and check all container will // be allocated in h1 - RMApp app1 = rm1.submitApp(200, "app", "user", null, "a1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // A has only 10% of x, so it can only allocate one container in label=empty @@ -268,7 +278,15 @@ public RMNodeLabelsManager createNodeLabelManager() { // launch an app to queue a1 (label = x), and check all container will // be allocated in h1 - RMApp app1 = rm1.submitApp(1024, "app", "user", null, "a1"); + MockRMAppSubmissionData data2 = + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data2); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // request a container (label = y). can be allocated on nm2 @@ -282,7 +300,15 @@ public RMNodeLabelsManager createNodeLabelManager() { // launch an app to queue b1 (label = y), and check all container will // be allocated in h5 - RMApp app2 = rm1.submitApp(1024, "app", "user", null, "b1"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data1); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm5); // request a container for AM, will succeed @@ -296,7 +322,15 @@ public RMNodeLabelsManager createNodeLabelManager() { RMContainerState.ALLOCATED)); // launch an app to queue b2 - RMApp app3 = rm1.submitApp(1024, "app", "user", null, "b2"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b2") + .withUnmanaged(false) + .build(); + RMApp app3 = MockRMAppSubmitter.submit(rm1, data); MockAM am3 = MockRM.launchAndRegisterAM(app3, rm1, nm5); // request a container. try to allocate on nm1 (label = x) and nm3 (label = @@ -347,7 +381,15 @@ public RMNodeLabelsManager createNodeLabelManager() { // launch an app to queue a1 (label = x), and check all container will // be allocated in h1 - RMApp app1 = rm1.submitApp(200, "app", "user", null, "a1"); + MockRMAppSubmissionData data2 = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data2); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm3); // request a container. @@ -363,7 +405,15 @@ public RMNodeLabelsManager createNodeLabelManager() { // launch an app to queue b1 (label = y), and check all container will // be allocated in h2 - RMApp app2 = rm1.submitApp(200, "app", "user", null, "b1"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data1); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm3); // request a container. @@ -378,7 +428,15 @@ public RMNodeLabelsManager createNodeLabelManager() { // launch an app to queue c1 (label = ""), and check all container will // be allocated in h3 - RMApp app3 = rm1.submitApp(200, "app", "user", null, "c1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c1") + .withUnmanaged(false) + .build(); + RMApp app3 = MockRMAppSubmitter.submit(rm1, data); MockAM am3 = MockRM.launchAndRegisterAM(app3, rm1, nm3); // request a container. @@ -423,7 +481,15 @@ public RMNodeLabelsManager createNodeLabelManager() { // launch an app to queue a1 (label = x), and check all container will // be allocated in h1 - RMApp app1 = rm1.submitApp(200, "app", "user", null, "a1"); + MockRMAppSubmissionData data2 = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data2); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // request a container. @@ -439,7 +505,15 @@ public RMNodeLabelsManager createNodeLabelManager() { // launch an app to queue b1 (label = y), and check all container will // be allocated in h2 - RMApp app2 = rm1.submitApp(200, "app", "user", null, "b1"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data1); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm2); // request a container. @@ -454,7 +528,15 @@ public RMNodeLabelsManager createNodeLabelManager() { // launch an app to queue c1 (label = ""), and check all container will // be allocated in h3 - RMApp app3 = rm1.submitApp(200, "app", "user", null, "c1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c1") + .withUnmanaged(false) + .build(); + RMApp app3 = MockRMAppSubmitter.submit(rm1, data); MockAM am3 = MockRM.launchAndRegisterAM(app3, rm1, nm3); // request a container. @@ -503,7 +585,15 @@ public RMNodeLabelsManager createNodeLabelManager() { // launch an app to queue a1 (label = x), and check all container will // be allocated in h1 - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "a1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // request a container. @@ -569,10 +659,26 @@ public void testRMContainerLeakInLeafQueue() throws Exception { // launch an app to queue a1 (label = x), and check all container will // be allocated in h1 - RMApp app1 = rm1.submitApp(1 * GB, "app1", "user", null, "a1"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app1") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); - RMApp app2 = rm1.submitApp(1 * GB, "app2", "user", null, "a1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app2") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data); MockRM.launchAndRegisterAM(app2, rm1, nm1); // request a container. @@ -654,7 +760,15 @@ public RMNodeLabelsManager createNodeLabelManager() { MockNM nm2 = rm1.registerNode("h2:1234", 40 * GB); // label = y // launch an app to queue b1 (label = y), AM container should be launched in // nm2 - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "b1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm2); // Creating request set when request before ANY is not having label and any // is having label @@ -752,11 +866,27 @@ public RMNodeLabelsManager createNodeLabelManager() { MockNM nm2 = rm1.registerNode("h2:1234", 100 * GB); // label = // launch an app to queue b1 (label = y), AM container should be launched in nm2 - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "b1"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm2); // launch another app to queue b1 (label = y), AM container should be launched in nm2 - RMApp app2 = rm1.submitApp(1 * GB, "app", "user", null, "b1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm2); // request container and nm1 do heartbeat (nm2 has label=y), note that app1 @@ -830,7 +960,15 @@ public RMNodeLabelsManager createNodeLabelManager() { MockNM nm2 = rm1.registerNode("h2:1234", 100 * GB); // label = // launch an app to queue b1 (label = y), AM container should be launched in nm3 - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "b1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm2); // request containers from am2, priority=1 asks for "" and priority=2 asks @@ -881,7 +1019,15 @@ public RMNodeLabelsManager createNodeLabelManager() { ContainerId nextContainerId; // launch an app to queue b1 (label = y), AM container should be launched in nm3 - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "b1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm2); // request containers from am2, priority=1 asks for "" * 6 (id from 4 to 9), @@ -999,27 +1145,75 @@ public RMNodeLabelsManager createNodeLabelManager() { MockNM nm2 = rm1.registerNode("h2:1234", 100 * GB); // label = // app1 -> a1 - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "a1"); + MockRMAppSubmissionData data5 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data5); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // app2 -> a2 - RMApp app2 = rm1.submitApp(1 * GB, "app", "user", null, "a2"); + MockRMAppSubmissionData data4 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a2") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data4); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm2); // app3 -> b1 - RMApp app3 = rm1.submitApp(1 * GB, "app", "user", null, "b1"); + MockRMAppSubmissionData data3 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + RMApp app3 = MockRMAppSubmitter.submit(rm1, data3); MockAM am3 = MockRM.launchAndRegisterAM(app3, rm1, nm1); // app4 -> b2 - RMApp app4 = rm1.submitApp(1 * GB, "app", "user", null, "b2"); + MockRMAppSubmissionData data2 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b2") + .withUnmanaged(false) + .build(); + RMApp app4 = MockRMAppSubmitter.submit(rm1, data2); MockAM am4 = MockRM.launchAndRegisterAM(app4, rm1, nm2); // app5 -> c1 - RMApp app5 = rm1.submitApp(1 * GB, "app", "user", null, "c1"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c1") + .withUnmanaged(false) + .build(); + RMApp app5 = MockRMAppSubmitter.submit(rm1, data1); MockAM am5 = MockRM.launchAndRegisterAM(app5, rm1, nm1); // app6 -> b2 - RMApp app6 = rm1.submitApp(1 * GB, "app", "user", null, "c2"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c2") + .withUnmanaged(false) + .build(); + RMApp app6 = MockRMAppSubmitter.submit(rm1, data); MockAM am6 = MockRM.launchAndRegisterAM(app6, rm1, nm2); // Each application request 5 * 1GB container @@ -1108,7 +1302,15 @@ public RMNodeLabelsManager createNodeLabelManager() { MockNM nm2 = rm1.registerNode("h2:1234", 10 * GB); // label = // app1 -> b - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "b"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm2); // Each application request 50 * 1GB container @@ -1172,7 +1374,15 @@ public RMNodeLabelsManager createNodeLabelManager() { MockNM nm1 = rm1.registerNode(nodeIdStr, 8 * GB); // label = x // launch an app to queue b1 (label = y), AM container should be launched in nm3 - RMApp app = rm1.submitApp(1 * GB, "app", "user", null, "b1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm1, data); CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler(); RMNode rmNode1 = rm1.getRMContext().getRMNodes().get(nm1.getNodeId()); @@ -1259,7 +1469,15 @@ public RMNodeLabelsManager createNodeLabelManager() { MockNM nm2 = rm1.registerNode("h2:1234", 10 * GB); // label = // app1 -> a - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "a"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm2); // app1 asks for 10 partition= containers @@ -1447,7 +1665,15 @@ public RMNodeLabelsManager createNodeLabelManager() { checkQueueUsedCapacity("root", cs, "", 0f, 0f); // app1 -> a1 - RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "a1"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm2); // app1 asks for 1 partition= containers @@ -1498,7 +1724,15 @@ public RMNodeLabelsManager createNodeLabelManager() { // submit an app to a2, uses 1 NON_PARTITIONED container and 1 PARTITIONED // container // app2 -> a2 - RMApp app2 = rm.submitApp(1 * GB, "app", "user", null, "a2"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a2") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm2); // app1 asks for 1 partition= containers @@ -1655,19 +1889,51 @@ public RMNodeLabelsManager createNodeLabelManager() { MockNM nm2 = rm.registerNode("h2:1234", 10 * GB); // label = // app1 -> a - RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "a"); + MockRMAppSubmissionData data3 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data3); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm2); // app2 -> b - RMApp app2 = rm.submitApp(1 * GB, "app", "user", null, "b"); + MockRMAppSubmissionData data2 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm, data2); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm2); // app3 -> c - RMApp app3 = rm.submitApp(1 * GB, "app", "user", null, "c"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c") + .withUnmanaged(false) + .build(); + RMApp app3 = MockRMAppSubmitter.submit(rm, data1); MockAM am3 = MockRM.launchAndRegisterAM(app3, rm, nm2); // app4 -> d - RMApp app4 = rm.submitApp(1 * GB, "app", "user", null, "d"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("d") + .withUnmanaged(false) + .build(); + RMApp app4 = MockRMAppSubmitter.submit(rm, data); MockAM am4 = MockRM.launchAndRegisterAM(app4, rm, nm2); // Test case 1 @@ -1817,11 +2083,27 @@ public RMNodeLabelsManager createNodeLabelManager() { MockNM nm1 = rm.registerNode("h1:1234", 10 * GB); // label = x // app1 -> a - RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "a", "x"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a") + .withAmLabel("x") + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); // app2 -> b - RMApp app2 = rm.submitApp(1 * GB, "app", "user", null, "b", "x"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b") + .withAmLabel("x") + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm1); // Both a/b has used_capacity(x) = 0, when doing exclusive allocation, a @@ -1909,7 +2191,15 @@ public RMNodeLabelsManager createNodeLabelManager() { nm1.registerNode(); // Launch app1 in a1, resource usage is 1GB (am) + 4GB * 2 = 9GB - RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "a1", "x"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withAmLabel("x") + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); am1.allocate("*", 4 * GB, 2, new ArrayList(), "x"); doNMHeartbeat(rm, nm1.getNodeId(), 10); @@ -1917,7 +2207,15 @@ public RMNodeLabelsManager createNodeLabelManager() { cs.getApplicationAttempt(am1.getApplicationAttemptId())); // Try to launch app2 in a2, asked 2GB, should success - RMApp app2 = rm.submitApp(2 * GB, "app", "user", null, "a2", "x"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(2 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a2") + .withAmLabel("x") + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm1); // am2 asks more resources, cannot success because current used = 9G (app1) @@ -1989,7 +2287,15 @@ public RMNodeLabelsManager createNodeLabelManager() { MockNM nm1 = rm1.registerNode("h1:1234", 10 * GB); // label = x MockNM nm2 = rm1.registerNode("h2:1234", 10 * GB); // label = y // app1 -> a - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "a", "x"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a") + .withAmLabel("x") + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // app1 asks for 5 partition=x containers @@ -2089,7 +2395,15 @@ public RMNodeLabelsManager createNodeLabelManager() { MockNM nm1 = rm1.registerNode("h1:1234", 10 * GB); // label = x MockNM nm2 = rm1.registerNode("h2:1234", 10 * GB); // label = // app1 -> a - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "a"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm2); // app1 asks for 3 partition= containers @@ -2225,7 +2539,15 @@ public RMNodeLabelsManager createNodeLabelManager() { rm1.start(); MockNM nm1 = rm1.registerNode("h1:1234", 20 * GB); // label = x // app1 -> a1 - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "a1", "x"); + MockRMAppSubmissionData data3 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withAmLabel("x") + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data3); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // app1 asks for 6 partition=x containers @@ -2245,7 +2567,15 @@ public RMNodeLabelsManager createNodeLabelManager() { // Try to launch app2 in a2, asked 2GB, should success // app2 -> a2 - RMApp app2 = rm1.submitApp(1 * GB, "app", "user", null, "a2", "x"); + MockRMAppSubmissionData data2 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a2") + .withAmLabel("x") + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data2); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1); // app2 asks for 4 partition=x containers @@ -2269,7 +2599,15 @@ public RMNodeLabelsManager createNodeLabelManager() { // Try to launch app3 in a2, asked 6GB, should fail // app3 -> a2 - RMApp app3 = rm1.submitApp(1 * GB, "app", "user", null, "a2", "x"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a2") + .withAmLabel("x") + .build(); + RMApp app3 = MockRMAppSubmitter.submit(rm1, data1); MockAM am3 = MockRM.launchAndRegisterAM(app3, rm1, nm1); am3.allocate("*", 1 * GB, 6, new ArrayList(), "x"); @@ -2289,7 +2627,15 @@ public RMNodeLabelsManager createNodeLabelManager() { rm1.waitForAppRemovedFromScheduler(app1.getApplicationId()); // app4 -> a1, try to allocate more than 6GB resource, should fail - RMApp app4 = rm1.submitApp(1 * GB, "app", "user", null, "a1", "x"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withAmLabel("x") + .build(); + RMApp app4 = MockRMAppSubmitter.submit(rm1, data); MockAM am4 = MockRM.launchAndRegisterAM(app4, rm1, nm1); // app3 asks for 7 partition=x containers diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueState.java index 0a39e99b38f..951155c9ee0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueState.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueState.java @@ -24,7 +24,7 @@ import static org.mockito.Mockito.when; import java.io.IOException; -import org.apache.hadoop.security.AccessControlException; + import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Priority; @@ -34,6 +34,8 @@ import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; @@ -227,7 +229,15 @@ public void testRecoverDrainingStateAfterRMRestart() throws Exception { MockNM nm = rm.registerNode("h1:1234", 204800); // submit an app, AM is running on nm1 - RMApp app = rm.submitApp(1024, "appname", "appuser", null, Q2); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm) + .withName("appname") + .withUser("appuser") + .withAcls(null) + .withQueue(Q2) + .withUnmanaged(false) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); MockRM.launchAM(app, rm, nm); rm.waitForState(app.getApplicationId(), RMAppState.ACCEPTED); // update queue state to STOPPED diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestSchedulingRequestContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestSchedulingRequestContainerAllocation.java index 26c709f12a5..1eabd6bc5e5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestSchedulingRequestContainerAllocation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestSchedulingRequestContainerAllocation.java @@ -24,8 +24,11 @@ import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; import org.apache.hadoop.yarn.api.records.ExecutionType; import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest; +import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.SchedulingRequest; import org.apache.hadoop.yarn.api.resource.PlacementConstraint; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.TargetApplicationsNamespace; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; @@ -54,6 +57,7 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -84,6 +88,21 @@ public void setUp() throws Exception { mgr.init(conf); } + private RMApp submitApp(MockRM rm, int memory, Set appTags) throws Exception { + Resource resource = Resource.newInstance(memory, 0); + ResourceRequest amResourceRequest = ResourceRequest.newInstance( + Priority.newInstance(0), ResourceRequest.ANY, resource, 1); + List amResourceRequests = + Collections.singletonList(amResourceRequest); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(resource, rm) + .withAmLabel(null) + .withAmResourceRequests(amResourceRequests) + .withApplicationTags(appTags) + .build(); + return MockRMAppSubmitter.submit(rm, data); + } + @Test public void testIntraAppAntiAffinity() throws Exception { Configuration csConf = TestUtils.getConfigurationWithMultipleQueues( @@ -111,7 +130,15 @@ public RMNodeLabelsManager createNodeLabelManager() { } // app1 -> c - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "c"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nms[0]); // app1 asks for 10 anti-affinity containers for the same app. It should @@ -194,7 +221,15 @@ public RMNodeLabelsManager createNodeLabelManager() { } // app1 -> c - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "c"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nms[0]); // app1 asks for 2 anti-affinity containers for the same app. @@ -287,7 +322,15 @@ public RMNodeLabelsManager createNodeLabelManager() { } // app1 -> c - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "c"); + MockRMAppSubmissionData data2 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data2); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nms[0]); // app1 asks for 3 anti-affinity containers for the same app. It should @@ -317,7 +360,15 @@ public RMNodeLabelsManager createNodeLabelManager() { Assert.assertEquals(4, schedulerApp.getLiveContainers().size()); // app2 -> c - RMApp app2 = rm1.submitApp(1 * GB, "app", "user", null, "c"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data1); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nms[0]); // App2 asks for 3 containers that anti-affinity with any mapper, @@ -353,7 +404,15 @@ public RMNodeLabelsManager createNodeLabelManager() { })); // app3 -> c - RMApp app3 = rm1.submitApp(1 * GB, "app", "user", null, "c"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c") + .withUnmanaged(false) + .build(); + RMApp app3 = MockRMAppSubmitter.submit(rm1, data); MockAM am3 = MockRM.launchAndRegisterAM(app3, rm1, nms[0]); // App3 asks for 3 containers that anti-affinity with any mapper. @@ -405,7 +464,15 @@ public RMNodeLabelsManager createNodeLabelManager() { } // app1 -> c - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "c"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nms[0]); // app1 asks for 2 anti-affinity containers for the same app. @@ -450,7 +517,15 @@ public RMNodeLabelsManager createNodeLabelManager() { } // app1 -> c - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "c"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nms[0]); CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler(); @@ -581,7 +656,7 @@ public void testInterAppCompositeConstraints() throws Exception { MockNM nm4 = rm.registerNode("192.168.0.4:1234", 100*GB, 100); MockNM nm5 = rm.registerNode("192.168.0.5:1234", 100*GB, 100); - RMApp app1 = rm.submitApp(1*GB, ImmutableSet.of("hbase")); + RMApp app1 = submitApp(rm, 1*GB, ImmutableSet.of("hbase")); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); // App1 (hbase) @@ -620,7 +695,7 @@ public void testInterAppCompositeConstraints() throws Exception { // App2 (web-server) // Web server instance has 2 instance and non of them can be co-allocated // with hbase-master. - RMApp app2 = rm.submitApp(1*GB, ImmutableSet.of("web-server")); + RMApp app2 = submitApp(rm, 1*GB, ImmutableSet.of("web-server")); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm2); // App2 (web-server) @@ -657,7 +732,7 @@ public void testInterAppCompositeConstraints() throws Exception { // App3 has multiple instances that must be co-allocated // with app2 server instance, and each node cannot have more than // 3 instances. - RMApp app3 = rm.submitApp(1*GB, ImmutableSet.of("ws-servants")); + RMApp app3 = submitApp(rm, 1*GB, ImmutableSet.of("ws-servants")); MockAM am3 = MockRM.launchAndRegisterAM(app3, rm, nm3); @@ -713,7 +788,7 @@ public void testMultiAllocationTagsConstraints() throws Exception { MockNM nm4 = rm.registerNode("192.168.0.4:1234", 10*GB, 10); MockNM nm5 = rm.registerNode("192.168.0.5:1234", 10*GB, 10); - RMApp app1 = rm.submitApp(1*GB, ImmutableSet.of("server1")); + RMApp app1 = submitApp(rm, 1*GB, ImmutableSet.of("server1")); // Allocate AM container on nm1 doNodeHeartbeat(nm1); RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); @@ -740,7 +815,7 @@ public void testMultiAllocationTagsConstraints() throws Exception { // App1 uses ports: 6000 String[] server2Ports = new String[] {"port_6000"}; - RMApp app2 = rm.submitApp(1*GB, ImmutableSet.of("server2")); + RMApp app2 = submitApp(rm, 1*GB, ImmutableSet.of("server2")); // Allocate AM container on nm1 doNodeHeartbeat(nm2); RMAppAttempt app2attempt1 = app2.getCurrentAppAttempt(); @@ -801,7 +876,7 @@ public void testInterAppConstraintsWithNamespaces() throws Exception { // App1 ~ app5 tag "former5" // App6 ~ app10 tag "latter5" String applicationTag = i<5 ? "former5" : "latter5"; - RMApp app = rm.submitApp(1*GB, ImmutableSet.of(applicationTag)); + RMApp app = submitApp(rm, 1*GB, ImmutableSet.of(applicationTag)); // Allocate AM container on nm1 doNodeHeartbeat(nm1, nm2, nm3, nm4, nm5); RMAppAttempt attempt = app.getCurrentAppAttempt(); @@ -828,7 +903,7 @@ public void testInterAppConstraintsWithNamespaces() throws Exception { // *** app-id // Submit another app, use app-id constraint against app5 - RMApp app1 = rm.submitApp(1*GB, ImmutableSet.of("xyz")); + RMApp app1 = submitApp(rm, 1*GB, ImmutableSet.of("xyz")); // Allocate AM container on nm1 doNodeHeartbeat(nm1); RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); @@ -858,7 +933,7 @@ public void testInterAppConstraintsWithNamespaces() throws Exception { } // *** app-tag - RMApp app2 = rm.submitApp(1*GB); + RMApp app2 = MockRMAppSubmitter.submitWithMemory(1 * GB, rm); // Allocate AM container on nm1 doNodeHeartbeat(nm2); RMAppAttempt app2attempt1 = app2.getCurrentAppAttempt(); @@ -883,7 +958,7 @@ public void testInterAppConstraintsWithNamespaces() throws Exception { } // *** not-self - RMApp app3 = rm.submitApp(1*GB); + RMApp app3 = MockRMAppSubmitter.submitWithMemory(1 * GB, rm); // Allocate AM container on nm1 doNodeHeartbeat(nm3); RMAppAttempt app3attempt1 = app3.getCurrentAppAttempt(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestSchedulingRequestContainerAllocationAsync.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestSchedulingRequestContainerAllocationAsync.java index d1d05dc407f..d172e3768c8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestSchedulingRequestContainerAllocationAsync.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestSchedulingRequestContainerAllocationAsync.java @@ -27,6 +27,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; @@ -86,7 +88,15 @@ public RMNodeLabelsManager createNodeLabelManager() { } // app1 -> c - RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "c"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nms[0]); // app1 asks for 10 anti-affinity containers for the same app. It should diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestWorkPreservingRMRestartForNodeLabel.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestWorkPreservingRMRestartForNodeLabel.java index 36ee68e80cb..62668a76334 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestWorkPreservingRMRestartForNodeLabel.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestWorkPreservingRMRestartForNodeLabel.java @@ -33,6 +33,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.TestRMRestart; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; @@ -154,7 +156,15 @@ public RMNodeLabelsManager createNodeLabelManager() { // launch an app to queue a1 (label = x), and check all container will // be allocated in h1 - RMApp app1 = rm1.submitApp(200, "app", "user", null, "a1"); + MockRMAppSubmissionData data2 = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data2); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // request a container. @@ -170,7 +180,15 @@ public RMNodeLabelsManager createNodeLabelManager() { // launch an app to queue b1 (label = y), and check all container will // be allocated in h2 - RMApp app2 = rm1.submitApp(200, "app", "user", null, "b1"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data1); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm2); // request a container. @@ -185,7 +203,15 @@ public RMNodeLabelsManager createNodeLabelManager() { // launch an app to queue c1 (label = ""), and check all container will // be allocated in h3 - RMApp app3 = rm1.submitApp(200, "app", "user", null, "c1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("c1") + .withUnmanaged(false) + .build(); + RMApp app3 = MockRMAppSubmitter.submit(rm1, data); MockAM am3 = MockRM.launchAndRegisterAM(app3, rm1, nm3); // request a container. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestPlacementProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestPlacementProcessor.java index e129a750c17..760ec3841a3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestPlacementProcessor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestPlacementProcessor.java @@ -38,6 +38,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; @@ -127,7 +129,15 @@ public void testAntiAffinityPlacement() throws Exception { nm3.registerNode(); nm4.registerNode(); - RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); // Containers with allocationTag 'foo' are restricted to 1 per NODE MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm2, Collections.singletonMap(Collections.singleton("foo"), @@ -176,7 +186,15 @@ public void testMutualAntiAffinityPlacement() throws Exception { nm4.registerNode(); nm5.registerNode(); - RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); // Containers with allocationTag 'foo' are restricted to 1 per NODE Map, PlacementConstraint> pcMap = new HashMap<>(); pcMap.put(Collections.singleton("foo"), @@ -227,7 +245,15 @@ public void testCardinalityPlacement() throws Exception { nm3.registerNode(); nm4.registerNode(); - RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); // Containers with allocationTag 'foo' should not exceed 4 per NODE MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm2, Collections.singletonMap(Collections.singleton("foo"), @@ -280,7 +306,15 @@ public void testAffinityPlacement() throws Exception { nm3.registerNode(); nm4.registerNode(); - RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); // Containers with allocationTag 'foo' should be placed where // containers with allocationTag 'bar' are already running MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm2, @@ -328,7 +362,15 @@ public void testComplexPlacement() throws Exception { nm3.registerNode(); nm4.registerNode(); - RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); Map, PlacementConstraint> constraintMap = new HashMap<>(); // Containers with allocationTag 'bar' should not exceed 1 per NODE constraintMap.put(Collections.singleton("bar"), @@ -399,7 +441,15 @@ public void testSchedulerRejection() throws Exception { nm3.registerNode(); nm4.registerNode(); - RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "a"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("a") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); // Containers with allocationTag 'foo' are restricted to 1 per NODE MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm2, Collections.singletonMap( @@ -470,7 +520,15 @@ public void testNodeCapacityRejection() throws Exception { nm3.registerNode(); nm4.registerNode(); - RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); // Containers with allocationTag 'foo' are restricted to 1 per NODE MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm2, Collections.singletonMap( @@ -556,7 +614,15 @@ public void testRePlacementAfterSchedulerRejection() throws Exception { nm4.registerNode(); // Do not register nm5 yet.. - RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); // Containers with allocationTag 'foo' are restricted to 1 per NODE MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm2, Collections.singletonMap( @@ -628,7 +694,15 @@ public void testPlacementRejection() throws Exception { nm3.registerNode(); nm4.registerNode(); - RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); // Containers with allocationTag 'foo' are restricted to 1 per NODE MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm2, Collections.singletonMap( @@ -703,7 +777,15 @@ public void testAndOrPlacement() throws Exception { nm3.registerNode(); nm4.registerNode(); - RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); // Register app1 with following constraints // 1) foo anti-affinity with foo on node diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestApplicationMasterServiceWithFS.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestApplicationMasterServiceWithFS.java index 2416d8cf9e5..179c01e86a5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestApplicationMasterServiceWithFS.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestApplicationMasterServiceWithFS.java @@ -33,6 +33,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; @@ -97,7 +99,10 @@ public void testQueueLevelContainerAllocationFail() throws Exception { MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * GB); // Submit an application - RMApp app1 = rm.submitApp(2 * GB, "queueA"); + MockRMAppSubmissionData data = MockRMAppSubmissionData.Builder.createWithDefaults(2 * GB, rm) + .withQueue("queueA") + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); // kick the scheduling nm1.nodeHeartbeat(true); @@ -139,7 +144,10 @@ private void testFairSchedulerContainerAllocationSuccess(String queueName) MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * GB); // Submit an application - RMApp app1 = rm.submitApp(2 * GB, queueName); + MockRMAppSubmissionData data = MockRMAppSubmissionData.Builder.createWithDefaults(2 * GB, rm) + .withQueue(queueName) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); // kick the scheduling nm1.nodeHeartbeat(true); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 0d6caebac68..19eb5e62d87 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -83,6 +83,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockNodes; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.NodeManager; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; @@ -5304,7 +5306,15 @@ public void testRefreshQueuesWhenRMHA() throws Exception { rm2.getAdminService().transitionToActive(requestInfo); // 4. submit a app to the new added queue "test_queue" - RMApp app = rm2.submitApp(200, "test_app", "user", null, "test_queue"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm2) + .withName("test_app") + .withUser("user") + .withAcls(null) + .withQueue("test_queue") + .withUnmanaged(false) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm2, data); RMAppAttempt attempt0 = app.getCurrentAppAttempt(); nm.nodeHeartbeat(true); MockAM am0 = rm2.sendAMLaunched(attempt0.getAppAttemptId()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java index ee66a49032f..c15f8bc9a31 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java @@ -66,6 +66,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockNodes; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.NodeManager; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; @@ -661,7 +662,7 @@ public void testAllocateContainerOnNodeWithoutOffSwitchSpecified() rm.start(); MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * GB); - RMApp app1 = rm.submitApp(2048); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(2048, rm); // kick the scheduling, 2 GB given to AM1, remaining 4GB on nm1 nm1.nodeHeartbeat(true); RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); @@ -693,7 +694,7 @@ public void testFifoScheduling() throws Exception { MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * GB); MockNM nm2 = rm.registerNode("127.0.0.2:5678", 4 * GB); - RMApp app1 = rm.submitApp(2048); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(2048, rm); // kick the scheduling, 2 GB given to AM1, remaining 4GB on nm1 nm1.nodeHeartbeat(true); RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); @@ -703,7 +704,7 @@ public void testFifoScheduling() throws Exception { rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemorySize()); - RMApp app2 = rm.submitApp(2048); + RMApp app2 = MockRMAppSubmitter.submitWithMemory(2048, rm); // kick the scheduling, 2GB given to AM, remaining 2 GB on nm2 nm2.nodeHeartbeat(true); RMAppAttempt attempt2 = app2.getCurrentAppAttempt(); @@ -813,7 +814,7 @@ private void testMinimumAllocation(YarnConfiguration conf, int testAlloc) MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * GB); // Submit an application - RMApp app1 = rm.submitApp(testAlloc); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(testAlloc, rm); // kick the scheduling nm1.nodeHeartbeat(true); @@ -1108,7 +1109,7 @@ public void testResourceOverCommit() throws Exception { MockNM nm1 = rm.registerNode("127.0.0.1:1234", 4 * GB); - RMApp app1 = rm.submitApp(2048); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(2048, rm); // kick the scheduling, 2 GB given to AM1, remaining 2GB on nm1 nm1.nodeHeartbeat(true); RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFairOrderingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFairOrderingPolicy.java index 683173af709..2820d1c5b5a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFairOrderingPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFairOrderingPolicy.java @@ -23,6 +23,8 @@ import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; @@ -172,10 +174,42 @@ public void testSizeBasedWeightNotAffectAppActivation() throws Exception { rm.registerNode("h1:1234", 10 * GB); // Submit 4 apps - rm.submitApp(1 * GB, "app", "user", null, "default"); - rm.submitApp(1 * GB, "app", "user", null, "default"); - rm.submitApp(1 * GB, "app", "user", null, "default"); - rm.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data3 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + MockRMAppSubmitter.submit(rm, data3); + MockRMAppSubmissionData data2 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + MockRMAppSubmitter.submit(rm, data2); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + MockRMAppSubmitter.submit(rm, data1); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + MockRMAppSubmitter.submit(rm, data); Assert.assertEquals(1, lq.getNumActiveApplications()); Assert.assertEquals(3, lq.getNumPendingApplications()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java index bcf239dfad6..49b93257409 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java @@ -56,6 +56,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.RMSecretManagerService; import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MockRMWithAMS; @@ -128,7 +129,7 @@ public void testTokenExpiry() throws Exception { try { MockNM nm1 = rm.registerNode("localhost:1234", 5120); - RMApp app = rm.submitApp(1024); + RMApp app = MockRMAppSubmitter.submitWithMemory(1024, rm); nm1.nodeHeartbeat(true); @@ -249,7 +250,7 @@ public void testMasterKeyRollOver() throws Exception { try { MockNM nm1 = rm.registerNode("localhost:1234", 5120); - RMApp app = rm.submitApp(1024); + RMApp app = MockRMAppSubmitter.submitWithMemory(1024, rm); nm1.nodeHeartbeat(true); @@ -379,7 +380,7 @@ protected AMRMTokenSecretManager createAMRMTokenSecretManager( }; rm.start(); MockNM nm = rm.registerNode("127.0.0.1:1234", 8000); - RMApp app = rm.submitApp(200); + RMApp app = MockRMAppSubmitter.submitWithMemory(200, rm); MockAM am = MockRM.launchAndRegisterAM(app, rm, nm); AMRMTokenSecretManager spySecretMgr = spySecretMgrRef.get(); // Do allocate. Should not update AMRMToken diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java index a0f4007cf8a..bb1eaeb6845 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java @@ -58,6 +58,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.MockRMWithCustomAMLauncher; import org.apache.hadoop.yarn.server.resourcemanager.ParameterizedSchedulerTestBase; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; @@ -216,7 +217,7 @@ protected void doSecureLogin() throws IOException { rm.start(); // Submit an app - RMApp app = rm.submitApp(1024); + RMApp app = MockRMAppSubmitter.submitWithMemory(1024, rm); // Set up a node. MockNM nm1 = rm.registerNode("localhost:1234", 3072); @@ -446,7 +447,7 @@ protected void doSecureLogin() throws IOException { rm.start(); // Submit an app - RMApp app = rm.submitApp(1024); + RMApp app = MockRMAppSubmitter.submitWithMemory(1024, rm); // Set up a node. MockNM nm1 = rm.registerNode("localhost:1234", 3072); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java index 9b2c0b327f1..b7dbe3f93b7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java @@ -85,6 +85,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.TestRMRestart.TestSecurityMockRM; import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore; @@ -345,6 +347,34 @@ static MyToken createTokens(Text renewer) token1.setService(new Text("localhost:0")); return token1; } + + public RMApp submitApp(MockRM rm, Credentials cred, ByteBuffer tokensConf) + throws Exception { + int maxAttempts = rm.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS); + MockRMAppSubmissionData data = MockRMAppSubmissionData.Builder.create() + .withResource(Resource.newInstance(200, 1)) + .withName("app1") + .withUser("user") + .withAcls(null) + .withUnmanaged(false) + .withQueue(null) + .withMaxAppAttempts(maxAttempts) + .withCredentials(cred) + .withAppType(null) + .withWaitForAccepted(true) + .withKeepContainers(false) + .withIsAppIdProvided(false) + .withApplicationId(null) + .withAttemptFailuresValidityInterval(0) + .withLogAggregationContext(null) + .withCancelTokensWhenComplete(true) + .withPriority(Priority.newInstance(0)) + .withAmLabel(null) + .withApplicationTimeouts(null) + .withTokensConf(tokensConf) + .build(); + return MockRMAppSubmitter.submit(rm, data); + } /** @@ -935,9 +965,16 @@ protected DelegationTokenRenewer createDelegationTokenRenewer() { credentials.addToken(userText1, token1); RMApp app = - rm.submitApp(200, "name", "user", - new HashMap(), false, "default", 1, - credentials); + MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm) + .withName("name") + .withUser("user") + .withAcls(new HashMap()) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(1) + .withCredentials(credentials) + .build()); // wait for the initial expiring hdfs token to be removed from allTokens GenericTestUtils.waitFor(new Supplier() { @@ -1010,9 +1047,16 @@ public void testRMRestartWithExpiredToken() throws Exception { MockRM rm1 = new TestSecurityMockRM(yarnConf); MemoryRMStateStore memStore = (MemoryRMStateStore) rm1.getRMStateStore(); rm1.start(); - RMApp app = rm1.submitApp(200, "name", "user", - new HashMap(), false, "default", 1, - credentials); + RMApp app = MockRMAppSubmitter.submit(rm1, + MockRMAppSubmissionData.Builder.createWithDefaults(200, rm1) + .withName("name") + .withUser("user") + .withAcls(new HashMap()) + .withUnmanaged(false) + .withQueue("default") + .withMaxAppAttempts(1) + .withCredentials(credentials) + .build()); // create token2 Text userText2 = new Text("user1"); @@ -1102,7 +1146,7 @@ protected DelegationTokenRenewer createDelegationTokenRenewer() { rm.start(); // submit an app without delegationToken - RMApp app = rm.submitApp(200); + RMApp app = MockRMAppSubmitter.submitWithMemory(200, rm); // wait for the new retrieved hdfs token. GenericTestUtils.waitFor(new Supplier() { @@ -1153,8 +1197,14 @@ public void testAppSubmissionWithPreviousToken() throws Exception{ // submit app1 with a token, set cancelTokenWhenComplete to false; Resource resource = Records.newRecord(Resource.class); resource.setMemorySize(200); - RMApp app1 = rm.submitApp(resource, "name", "user", null, false, null, 2, - credentials, null, true, false, false, null, 0, null, false, null); + MockRMAppSubmissionData data1 = MockRMAppSubmissionData.Builder.createWithDefaults(resource, rm) + .withName("name") + .withUser("user") + .withMaxAppAttempts(2) + .withCredentials(credentials) + .withCancelTokensWhenComplete(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); rm.waitForState(app1.getApplicationId(), RMAppState.RUNNING); DelegationTokenRenewer renewer = @@ -1163,8 +1213,14 @@ public void testAppSubmissionWithPreviousToken() throws Exception{ Assert.assertNotNull(dttr); // submit app2 with the same token, set cancelTokenWhenComplete to true; - RMApp app2 = rm.submitApp(resource, "name", "user", null, false, null, 2, - credentials, null, true, false, false, null, 0, null, true, null); + MockRMAppSubmissionData data = MockRMAppSubmissionData.Builder.createWithDefaults(resource, rm) + .withName("name") + .withUser("user") + .withMaxAppAttempts(2) + .withCredentials(credentials) + .withCancelTokensWhenComplete(true) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm1); rm.waitForState(app2.getApplicationId(), RMAppState.RUNNING); finishAMAndWaitForComplete(app2, rm, nm1, am2, dttr); @@ -1219,17 +1275,28 @@ public void testCancelWithMultipleAppSubmissions() throws Exception{ Resource resource = Records.newRecord(Resource.class); resource.setMemorySize(200); + MockRMAppSubmissionData data2 = MockRMAppSubmissionData.Builder.createWithDefaults(resource, rm) + .withName("name") + .withUser("user") + .withMaxAppAttempts(2) + .withCredentials(credentials) + .build(); RMApp app1 = - rm.submitApp(resource, "name", "user", null, false, null, 2, credentials, - null, true, false, false, null, 0, null, true, null); + MockRMAppSubmitter.submit(rm, data2); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); rm.waitForState(app1.getApplicationId(), RMAppState.RUNNING); DelegationTokenToRenew dttr = renewer.getAllTokens().get(token1); Assert.assertNotNull(dttr); Assert.assertTrue(dttr.referringAppIds.contains(app1.getApplicationId())); - RMApp app2 = rm.submitApp(resource, "name", "user", null, false, null, 2, - credentials, null, true, false, false, null, 0, null, true, null); + MockRMAppSubmissionData data1 = MockRMAppSubmissionData.Builder.createWithDefaults(resource, rm) + .withResource(resource) + .withName("name") + .withUser("user") + .withMaxAppAttempts(2) + .withCredentials(credentials) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm, data1); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm1); rm.waitForState(app2.getApplicationId(), RMAppState.RUNNING); Assert.assertTrue(renewer.getAllTokens().containsKey(token1)); @@ -1245,8 +1312,14 @@ public void testCancelWithMultipleAppSubmissions() throws Exception{ Assert.assertFalse(dttr.isTimerCancelled()); Assert.assertFalse(Renewer.cancelled); - RMApp app3 = rm.submitApp(resource, "name", "user", null, false, null, 2, - credentials, null, true, false, false, null, 0, null, true, null); + MockRMAppSubmissionData data = MockRMAppSubmissionData.Builder.createWithDefaults(resource, rm) + .withResource(resource) + .withName("name") + .withUser("user") + .withMaxAppAttempts(2) + .withCredentials(credentials) + .build(); + RMApp app3 = MockRMAppSubmitter.submit(rm, data); MockAM am3 = MockRM.launchAndRegisterAM(app3, rm, nm1); rm.waitForState(app3.getApplicationId(), RMAppState.RUNNING); Assert.assertTrue(renewer.getAllTokens().containsKey(token1)); @@ -1339,7 +1412,7 @@ public void testRenewTokenUsingTokenConfProvidedByApp() throws Exception{ final int confSize = appConf.size(); // submit app - RMApp app = rm.submitApp(credentials, tokenConf); + RMApp app = submitApp(rm, credentials, tokenConf); GenericTestUtils.waitFor(new Supplier() { public Boolean get() { @@ -1394,7 +1467,7 @@ public void testTokensConfExceedLimit() throws Exception { ByteBuffer tokenConf = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); try { - rm.submitApp(credentials, tokenConf); + submitApp(rm, credentials, tokenConf); Assert.fail(); } catch (Exception e) { e.printStackTrace(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/volume/csi/TestVolumeProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/volume/csi/TestVolumeProcessor.java index cee8fdf083c..c288c699734 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/volume/csi/TestVolumeProcessor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/volume/csi/TestVolumeProcessor.java @@ -33,6 +33,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; @@ -140,7 +142,15 @@ private void writeTmpResourceTypesFile(File tmpFile) throws IOException { @Test (timeout = 10000L) public void testVolumeProvisioning() throws Exception { - RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, mockNMS[0]); Resource resource = Resource.newInstance(1024, 1); ResourceInformation volumeResource = ResourceInformation @@ -193,7 +203,15 @@ public void testVolumeProvisioning() throws Exception { @Test (timeout = 30000L) public void testInvalidRequest() throws Exception { - RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, mockNMS[0]); Resource resource = Resource.newInstance(1024, 1); ResourceInformation volumeResource = ResourceInformation @@ -227,7 +245,15 @@ public void testInvalidRequest() throws Exception { @Test (timeout = 30000L) public void testProvisioningFailures() throws Exception { - RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1 * GB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, mockNMS[0]); CsiAdaptorProtocol mockedClient = Mockito diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServiceAppsNodelabel.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServiceAppsNodelabel.java index 3815fb9d06c..54566097040 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServiceAppsNodelabel.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServiceAppsNodelabel.java @@ -35,6 +35,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; @@ -145,7 +147,7 @@ public void testAppsFinished() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); amNodeManager.nodeHeartbeat(true); - RMApp killedApp = rm.submitApp(AM_CONTAINER_MB); + RMApp killedApp = MockRMAppSubmitter.submitWithMemory(AM_CONTAINER_MB, rm); rm.killApp(killedApp.getApplicationId()); WebResource r = resource(); ClientResponse response = @@ -173,7 +175,15 @@ public void testAppsRunning() throws JSONException, Exception { nodeLabelManager.addLabelsToNode( ImmutableMap.of(NodeId.newInstance("h2", 1235), toSet("X"))); - RMApp app1 = rm.submitApp(AM_CONTAINER_MB, "app", "user", null, "default"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(AM_CONTAINER_MB, rm) + .withName("app") + .withUser("user") + .withAcls(null) + .withQueue("default") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); nm1.nodeHeartbeat(true); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppAttempts.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppAttempts.java index a84fdcad934..649c9370d27 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppAttempts.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppAttempts.java @@ -30,6 +30,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; @@ -110,7 +112,12 @@ public TestRMWebServicesAppAttempts() { public void testAppAttempts() throws Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(CONTAINER_MB, rm) + .withName("testwordcount") + .withUser("user1") + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); amNodeManager.nodeHeartbeat(true); testAppAttemptsHelper(app1.getApplicationId().toString(), app1, MediaType.APPLICATION_JSON); @@ -121,7 +128,12 @@ public void testAppAttempts() throws Exception { public void testMultipleAppAttempts() throws Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 8192); - RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(CONTAINER_MB, rm) + .withName("testwordcount") + .withUser("user1") + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); MockAM am = MockRM.launchAndRegisterAM(app1, rm, amNodeManager); int maxAppAttempts = rm.getConfig().getInt( YarnConfiguration.RM_AM_MAX_ATTEMPTS, @@ -153,7 +165,7 @@ public void testMultipleAppAttempts() throws Exception { public void testAppAttemptsSlash() throws Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(CONTAINER_MB); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); amNodeManager.nodeHeartbeat(true); testAppAttemptsHelper(app1.getApplicationId().toString() + "/", app1, MediaType.APPLICATION_JSON); @@ -164,7 +176,7 @@ public void testAppAttemptsSlash() throws Exception { public void testAppAttemptsDefault() throws Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(CONTAINER_MB); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); amNodeManager.nodeHeartbeat(true); testAppAttemptsHelper(app1.getApplicationId().toString() + "/", app1, ""); rm.stop(); @@ -174,7 +186,7 @@ public void testAppAttemptsDefault() throws Exception { public void testInvalidAppIdGetAttempts() throws Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app = rm.submitApp(CONTAINER_MB); + RMApp app = MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); @@ -215,7 +227,7 @@ public void testInvalidAppIdGetAttempts() throws Exception { public void testInvalidAppAttemptId() throws Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app = rm.submitApp(CONTAINER_MB); + RMApp app = MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); @@ -256,7 +268,12 @@ public void testInvalidAppAttemptId() throws Exception { public void testNonexistAppAttempts() throws Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - rm.submitApp(CONTAINER_MB, "testwordcount", "user1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(CONTAINER_MB, rm) + .withName("testwordcount") + .withUser("user1") + .build(); + MockRMAppSubmitter.submit(rm, data); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); @@ -322,7 +339,12 @@ public void testAppAttemptsXML() throws Exception { rm.start(); String user = "user1"; MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", user); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(CONTAINER_MB, rm) + .withName("testwordcount") + .withUser(user) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("cluster") diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppCustomResourceTypes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppCustomResourceTypes.java index ab4dcf889cd..5ffee168cc2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppCustomResourceTypes.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppCustomResourceTypes.java @@ -27,6 +27,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; @@ -114,7 +116,12 @@ private WebResource getWebResourcePathForApp(RMApp app1, WebResource r) { public void testRunningAppXml() throws Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(CONTAINER_MB, rm) + .withName("testwordcount") + .withUser("user1") + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, amNodeManager); am1.allocate("*", 2048, 1, new ArrayList<>()); amNodeManager.nodeHeartbeat(true); @@ -142,7 +149,12 @@ public void testRunningAppXml() throws Exception { public void testRunningAppJson() throws Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(CONTAINER_MB, rm) + .withName("testwordcount") + .withUser("user1") + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, amNodeManager); am1.allocate("*", 2048, 1, new ArrayList<>()); amNodeManager.nodeHeartbeat(true); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index 6541eecf364..ad4e58db87c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -39,6 +39,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; @@ -131,7 +133,7 @@ public TestRMWebServicesApps() { public void testApps() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(CONTAINER_MB); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); amNodeManager.nodeHeartbeat(true); testAppsHelper("apps", app1, MediaType.APPLICATION_JSON); rm.stop(); @@ -141,7 +143,7 @@ public void testApps() throws JSONException, Exception { public void testAppsSlash() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(CONTAINER_MB); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); amNodeManager.nodeHeartbeat(true); testAppsHelper("apps/", app1, MediaType.APPLICATION_JSON); rm.stop(); @@ -151,7 +153,7 @@ public void testAppsSlash() throws JSONException, Exception { public void testAppsDefault() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(CONTAINER_MB); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); amNodeManager.nodeHeartbeat(true); testAppsHelper("apps/", app1, ""); rm.stop(); @@ -161,7 +163,12 @@ public void testAppsDefault() throws JSONException, Exception { public void testAppsXML() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(CONTAINER_MB, rm) + .withName("testwordcount") + .withUser("user1") + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("cluster") @@ -187,7 +194,12 @@ public void testAppsXML() throws JSONException, Exception { public void testRunningApp() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(CONTAINER_MB, rm) + .withName("testwordcount") + .withUser("user1") + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, amNodeManager); am1.allocate("*", 4096, 1, new ArrayList<>()); amNodeManager.nodeHeartbeat(true); @@ -218,8 +230,18 @@ public void testRunningApp() throws JSONException, Exception { public void testAppsXMLMulti() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - rm.submitApp(CONTAINER_MB, "testwordcount", "user1"); - rm.submitApp(2048, "testwordcount2", "user1"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(CONTAINER_MB, rm) + .withName("testwordcount") + .withUser("user1") + .build(); + MockRMAppSubmitter.submit(rm, data1); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(2048, rm) + .withName("testwordcount2") + .withUser("user1") + .build(); + MockRMAppSubmitter.submit(rm, data); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); @@ -269,7 +291,7 @@ public void testAppsHelper(String path, RMApp app, String media, public void testAppsQueryState() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(CONTAINER_MB); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); @@ -293,8 +315,8 @@ public void testAppsQueryState() throws JSONException, Exception { public void testAppsQueryStates() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - rm.submitApp(CONTAINER_MB); - RMApp killedApp = rm.submitApp(CONTAINER_MB); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); + RMApp killedApp = MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); rm.killApp(killedApp.getApplicationId()); amNodeManager.nodeHeartbeat(true); @@ -344,8 +366,8 @@ public void testAppsQueryStates() throws JSONException, Exception { public void testAppsQueryStatesComma() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - rm.submitApp(CONTAINER_MB); - RMApp killedApp = rm.submitApp(CONTAINER_MB); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); + RMApp killedApp = MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); rm.killApp(killedApp.getApplicationId()); amNodeManager.nodeHeartbeat(true); @@ -395,7 +417,7 @@ public void testAppsQueryStatesComma() throws JSONException, Exception { public void testAppsQueryStatesNone() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - rm.submitApp(CONTAINER_MB); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); @@ -416,7 +438,7 @@ public void testAppsQueryStatesNone() throws JSONException, Exception { public void testAppsQueryStateNone() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - rm.submitApp(CONTAINER_MB); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); @@ -437,7 +459,7 @@ public void testAppsQueryStateNone() throws JSONException, Exception { public void testAppsQueryStatesInvalid() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - rm.submitApp(CONTAINER_MB); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); @@ -475,7 +497,7 @@ public void testAppsQueryStatesInvalid() throws JSONException, Exception { public void testAppsQueryStateInvalid() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - rm.submitApp(CONTAINER_MB); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); @@ -513,7 +535,7 @@ public void testAppsQueryStateInvalid() throws JSONException, Exception { public void testAppsQueryFinalStatus() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(CONTAINER_MB); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); @@ -538,7 +560,7 @@ public void testAppsQueryFinalStatus() throws JSONException, Exception { public void testAppsQueryFinalStatusNone() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - rm.submitApp(CONTAINER_MB); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); @@ -558,7 +580,7 @@ public void testAppsQueryFinalStatusNone() throws JSONException, Exception { public void testAppsQueryFinalStatusInvalid() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - rm.submitApp(CONTAINER_MB); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); @@ -597,8 +619,8 @@ public void testAppsQueryFinalStatusInvalid() throws JSONException, Exception { public void testAppsQueryUser() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - rm.submitApp(CONTAINER_MB); - rm.submitApp(CONTAINER_MB); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); @@ -626,8 +648,8 @@ public void testAppsQueryUser() throws JSONException, Exception { public void testAppsQueryQueue() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - rm.submitApp(CONTAINER_MB); - rm.submitApp(CONTAINER_MB); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); @@ -650,8 +672,8 @@ public void testAppsQueryQueue() throws JSONException, Exception { public void testAppsQueryQueueAndStateTwoFinishedApps() throws Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(CONTAINER_MB); - RMApp app2 = rm.submitApp(CONTAINER_MB); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); + RMApp app2 = MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); amNodeManager.nodeHeartbeat(true); finishApp(amNodeManager, app1); @@ -686,8 +708,8 @@ public void testAppsQueryQueueAndStateTwoFinishedApps() throws Exception { public void testAppsQueryQueueAndStateOneFinishedApp() throws Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp finishedApp = rm.submitApp(CONTAINER_MB); - RMApp runningApp = rm.submitApp(CONTAINER_MB); + RMApp finishedApp = MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); + RMApp runningApp = MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); amNodeManager.nodeHeartbeat(true); finishApp(amNodeManager, finishedApp); @@ -722,8 +744,8 @@ public void testAppsQueryQueueAndStateOneFinishedApp() throws Exception { public void testAppsQueryQueueOneFinishedApp() throws Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp finishedApp = rm.submitApp(CONTAINER_MB); - RMApp runningApp = rm.submitApp(CONTAINER_MB); + RMApp finishedApp = MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); + RMApp runningApp = MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); amNodeManager.nodeHeartbeat(true); finishApp(amNodeManager, finishedApp); @@ -757,9 +779,9 @@ public void testAppsQueryQueueOneFinishedApp() throws Exception { public void testAppsQueryLimit() throws JSONException, Exception { rm.start(); rm.registerNode("127.0.0.1:1234", 2048); - rm.submitApp(CONTAINER_MB); - rm.submitApp(CONTAINER_MB); - rm.submitApp(CONTAINER_MB); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("cluster") .path("apps").queryParam("limit", "2") @@ -781,9 +803,9 @@ public void testAppsQueryStartBegin() throws JSONException, Exception { long start = System.currentTimeMillis(); Thread.sleep(1); rm.registerNode("127.0.0.1:1234", 2048); - rm.submitApp(CONTAINER_MB); - rm.submitApp(CONTAINER_MB); - rm.submitApp(CONTAINER_MB); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("cluster") .path("apps").queryParam("startedTimeBegin", String.valueOf(start)) @@ -803,11 +825,11 @@ public void testAppsQueryStartBegin() throws JSONException, Exception { public void testAppsQueryStartBeginSome() throws JSONException, Exception { rm.start(); rm.registerNode("127.0.0.1:1234", 2048); - rm.submitApp(CONTAINER_MB); - rm.submitApp(CONTAINER_MB); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); long start = System.currentTimeMillis(); Thread.sleep(1); - rm.submitApp(CONTAINER_MB); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("cluster") .path("apps").queryParam("startedTimeBegin", String.valueOf(start)) @@ -829,9 +851,9 @@ public void testAppsQueryStartEnd() throws JSONException, Exception { rm.registerNode("127.0.0.1:1234", 2048); long end = System.currentTimeMillis(); Thread.sleep(1); - rm.submitApp(CONTAINER_MB); - rm.submitApp(CONTAINER_MB); - rm.submitApp(CONTAINER_MB); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("cluster") .path("apps").queryParam("startedTimeEnd", String.valueOf(end)) @@ -851,11 +873,11 @@ public void testAppsQueryStartBeginEnd() throws JSONException, Exception { rm.registerNode("127.0.0.1:1234", 2048); long start = System.currentTimeMillis(); Thread.sleep(1); - rm.submitApp(CONTAINER_MB); - rm.submitApp(CONTAINER_MB); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); long end = System.currentTimeMillis(); Thread.sleep(1); - rm.submitApp(CONTAINER_MB); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("cluster") .path("apps").queryParam("startedTimeBegin", String.valueOf(start)) @@ -878,11 +900,11 @@ public void testAppsQueryFinishBegin() throws JSONException, Exception { MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); long start = System.currentTimeMillis(); Thread.sleep(1); - RMApp app1 = rm.submitApp(CONTAINER_MB); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); amNodeManager.nodeHeartbeat(true); finishApp(amNodeManager, app1); - rm.submitApp(CONTAINER_MB); - rm.submitApp(CONTAINER_MB); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("cluster") @@ -913,13 +935,13 @@ private void finishApp(MockNM amNodeManager, RMApp app) throws Exception { public void testAppsQueryFinishEnd() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(CONTAINER_MB); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); amNodeManager.nodeHeartbeat(true); // finish App finishApp(amNodeManager, app1); - rm.submitApp(CONTAINER_MB); - rm.submitApp(CONTAINER_MB); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); long end = System.currentTimeMillis(); WebResource r = resource(); @@ -943,13 +965,13 @@ public void testAppsQueryFinishBeginEnd() throws JSONException, Exception { MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); long start = System.currentTimeMillis(); Thread.sleep(1); - RMApp app1 = rm.submitApp(CONTAINER_MB); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); amNodeManager.nodeHeartbeat(true); // finish App finishApp(amNodeManager, app1); - rm.submitApp(CONTAINER_MB); - rm.submitApp(CONTAINER_MB); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); long end = System.currentTimeMillis(); WebResource r = resource(); @@ -973,15 +995,35 @@ public void testAppsQueryAppTypes() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); Thread.sleep(1); - RMApp app1 = rm.submitApp(CONTAINER_MB); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); amNodeManager.nodeHeartbeat(true); // finish App finishApp(amNodeManager, app1); - rm.submitApp(CONTAINER_MB, "", UserGroupInformation.getCurrentUser() - .getShortUserName(), null, false, null, 2, null, "MAPREDUCE"); - rm.submitApp(CONTAINER_MB, "", UserGroupInformation.getCurrentUser() - .getShortUserName(), null, false, null, 2, null, "NON-YARN"); + MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(CONTAINER_MB, rm) + .withName("") + .withUser(UserGroupInformation.getCurrentUser() + .getShortUserName()) + .withAcls(null) + .withUnmanaged(false) + .withQueue(null) + .withMaxAppAttempts(2) + .withCredentials(null) + .withAppType("MAPREDUCE") + .build()); + MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(CONTAINER_MB, rm) + .withName("") + .withUser(UserGroupInformation.getCurrentUser() + .getShortUserName()) + .withAcls(null) + .withUnmanaged(false) + .withQueue(null) + .withMaxAppAttempts(2) + .withCredentials(null) + .withAppType("NON-YARN") + .build()); WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("cluster") @@ -1164,7 +1206,7 @@ public void testAppsQueryWithInvaildDeselects() try { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - rm.submitApp(CONTAINER_MB); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("cluster") @@ -1197,7 +1239,7 @@ public void testAppsQueryWithDeselects() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - rm.submitApp(CONTAINER_MB); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); @@ -1303,15 +1345,45 @@ public void testAppStatistics() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 4096); Thread.sleep(1); - RMApp app1 = rm.submitApp(CONTAINER_MB, "", UserGroupInformation.getCurrentUser() - .getShortUserName(), null, false, null, 2, null, "MAPREDUCE"); + RMApp app1 = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(CONTAINER_MB, rm) + .withName("") + .withUser(UserGroupInformation.getCurrentUser() + .getShortUserName()) + .withAcls(null) + .withUnmanaged(false) + .withQueue(null) + .withMaxAppAttempts(2) + .withCredentials(null) + .withAppType("MAPREDUCE") + .build()); amNodeManager.nodeHeartbeat(true); finishApp(amNodeManager, app1); - rm.submitApp(CONTAINER_MB, "", UserGroupInformation.getCurrentUser() - .getShortUserName(), null, false, null, 2, null, "MAPREDUCE"); - rm.submitApp(CONTAINER_MB, "", UserGroupInformation.getCurrentUser() - .getShortUserName(), null, false, null, 2, null, "OTHER"); + MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(CONTAINER_MB, rm) + .withName("") + .withUser(UserGroupInformation.getCurrentUser() + .getShortUserName()) + .withAcls(null) + .withUnmanaged(false) + .withQueue(null) + .withMaxAppAttempts(2) + .withCredentials(null) + .withAppType("MAPREDUCE") + .build()); + MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithDefaults(CONTAINER_MB, rm) + .withName("") + .withUser(UserGroupInformation.getCurrentUser() + .getShortUserName()) + .withAcls(null) + .withUnmanaged(false) + .withQueue(null) + .withMaxAppAttempts(2) + .withCredentials(null) + .withAppType("OTHER") + .build()); // zero type, zero state WebResource r = resource(); @@ -1464,7 +1536,12 @@ public void testAppStatistics() throws JSONException, Exception { public void testSingleApp() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(CONTAINER_MB, rm) + .withName("testwordcount") + .withUser("user1") + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); amNodeManager.nodeHeartbeat(true); testSingleAppsHelper(app1.getApplicationId().toString(), app1, MediaType.APPLICATION_JSON); @@ -1475,7 +1552,12 @@ public void testSingleApp() throws JSONException, Exception { public void testUnmarshalAppInfo() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(CONTAINER_MB, rm) + .withName("testwordcount") + .withUser("user1") + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); @@ -1499,7 +1581,7 @@ public void testUnmarshalAppInfo() throws JSONException, Exception { public void testSingleAppsSlash() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(CONTAINER_MB); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); amNodeManager.nodeHeartbeat(true); testSingleAppsHelper(app1.getApplicationId().toString() + "/", app1, MediaType.APPLICATION_JSON); @@ -1510,7 +1592,7 @@ public void testSingleAppsSlash() throws JSONException, Exception { public void testSingleAppsDefault() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(CONTAINER_MB); + RMApp app1 = MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); amNodeManager.nodeHeartbeat(true); testSingleAppsHelper(app1.getApplicationId().toString() + "/", app1, ""); rm.stop(); @@ -1520,7 +1602,7 @@ public void testSingleAppsDefault() throws JSONException, Exception { public void testInvalidApp() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - rm.submitApp(CONTAINER_MB); + MockRMAppSubmitter.submitWithMemory(CONTAINER_MB, rm); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); @@ -1559,7 +1641,12 @@ public void testInvalidApp() throws JSONException, Exception { public void testNonexistApp() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - rm.submitApp(CONTAINER_MB, "testwordcount", "user1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(CONTAINER_MB, rm) + .withName("testwordcount") + .withUser("user1") + .build(); + MockRMAppSubmitter.submit(rm, data); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); @@ -1610,7 +1697,12 @@ public void testSingleAppsHelper(String path, RMApp app, String media) public void testSingleAppsXML() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(CONTAINER_MB, rm) + .withName("testwordcount") + .withUser("user1") + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("cluster") diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsCustomResourceTypes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsCustomResourceTypes.java index bcdb7138b98..b0b1cb645c8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsCustomResourceTypes.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsCustomResourceTypes.java @@ -29,6 +29,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; @@ -116,7 +118,12 @@ public TestRMWebServicesAppsCustomResourceTypes() { public void testRunningAppsXml() throws Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(CONTAINER_MB, rm) + .withName("testwordcount") + .withUser("user1") + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, amNodeManager); am1.allocate("*", 2048, 1, new ArrayList<>()); amNodeManager.nodeHeartbeat(true); @@ -147,7 +154,12 @@ public void testRunningAppsXml() throws Exception { public void testRunningAppsJson() throws Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(CONTAINER_MB, rm) + .withName("testwordcount") + .withUser("user1") + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, amNodeManager); am1.allocate("*", 2048, 1, new ArrayList<>()); amNodeManager.nodeHeartbeat(true); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java index 5d99e0d169d..5c6f996ae89 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java @@ -74,6 +74,8 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; @@ -339,7 +341,12 @@ public void testSingleAppState() throws Exception { String[] mediaTypes = { MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }; for (String mediaType : mediaTypes) { - RMApp app = rm.submitApp(CONTAINER_MB, "", webserviceUserName); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(CONTAINER_MB, rm) + .withName("") + .withUser(webserviceUserName) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); amNodeManager.nodeHeartbeat(true); ClientResponse response = this @@ -366,7 +373,12 @@ public void testSingleAppKill() throws Exception { String diagnostic = "message1"; for (String mediaType : mediaTypes) { for (MediaType contentType : contentTypes) { - RMApp app = rm.submitApp(CONTAINER_MB, "", webserviceUserName); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(CONTAINER_MB, rm) + .withName("") + .withUser(webserviceUserName) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); amNodeManager.nodeHeartbeat(true); AppState targetState = @@ -457,7 +469,12 @@ public void testSingleAppKillInvalidState() throws Exception { for (String mediaType : mediaTypes) { for (MediaType contentType : contentTypes) { for (String targetStateString : targetStates) { - RMApp app = rm.submitApp(CONTAINER_MB, "", webserviceUserName); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(CONTAINER_MB, rm) + .withName("") + .withUser(webserviceUserName) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); amNodeManager.nodeHeartbeat(true); ClientResponse response; AppState targetState = new AppState(targetStateString); @@ -564,7 +581,12 @@ public void testSingleAppKillUnauthorized() throws Exception { String[] mediaTypes = { MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }; for (String mediaType : mediaTypes) { - RMApp app = rm.submitApp(CONTAINER_MB, "test", "someuser"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(CONTAINER_MB, rm) + .withName("test") + .withUser("someuser") + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); amNodeManager.nodeHeartbeat(true); ClientResponse response = this @@ -1036,7 +1058,12 @@ public void testGetAppQueue() throws Exception { String[] contentTypes = { MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }; for (String contentType : contentTypes) { - RMApp app = rm.submitApp(CONTAINER_MB, "", webserviceUserName); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(CONTAINER_MB, rm) + .withName("") + .withUser(webserviceUserName) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); amNodeManager.nodeHeartbeat(true); ClientResponse response = this @@ -1090,7 +1117,12 @@ public void testUpdateAppPriority() throws Exception { { MediaType.APPLICATION_JSON_TYPE, MediaType.APPLICATION_XML_TYPE }; for (String mediaType : mediaTypes) { for (MediaType contentType : contentTypes) { - RMApp app = rm.submitApp(CONTAINER_MB, "", webserviceUserName); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(CONTAINER_MB, rm) + .withName("") + .withUser(webserviceUserName) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data1); amNodeManager.nodeHeartbeat(true); int modifiedPriority = 8; AppPriority priority = new AppPriority(modifiedPriority); @@ -1130,7 +1162,12 @@ public void testUpdateAppPriority() throws Exception { } // check unauthorized - app = rm.submitApp(CONTAINER_MB, "", "someuser"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(CONTAINER_MB, rm) + .withName("") + .withUser("someuser") + .build(); + app = MockRMAppSubmitter.submit(rm, data); amNodeManager.nodeHeartbeat(true); response = this .constructWebResource("apps", app.getApplicationId().toString(), @@ -1171,7 +1208,12 @@ public void testAppMove() throws Exception { { MediaType.APPLICATION_JSON_TYPE, MediaType.APPLICATION_XML_TYPE }; for (String mediaType : mediaTypes) { for (MediaType contentType : contentTypes) { - RMApp app = rm.submitApp(CONTAINER_MB, "", webserviceUserName); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(CONTAINER_MB, rm) + .withName("") + .withUser(webserviceUserName) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data1); amNodeManager.nodeHeartbeat(true); AppQueue targetQueue = new AppQueue("test"); Object entity; @@ -1204,7 +1246,12 @@ public void testAppMove() throws Exception { Assert.assertEquals(expectedQueue, app.getQueue()); // check unauthorized - app = rm.submitApp(CONTAINER_MB, "", "someuser"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(CONTAINER_MB, rm) + .withName("") + .withUser("someuser") + .build(); + app = MockRMAppSubmitter.submit(rm, data); amNodeManager.nodeHeartbeat(true); response = this @@ -1312,7 +1359,12 @@ public void testUpdateAppTimeout() throws Exception { for (String mediaType : mediaTypes) { for (MediaType contentType : contentTypes) { // application submitted without timeout - RMApp app = rm.submitApp(CONTAINER_MB, "", webserviceUserName); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(CONTAINER_MB, rm) + .withName("") + .withUser(webserviceUserName) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); ClientResponse response = this.constructWebResource("apps", app.getApplicationId().toString(), diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java index 46d0a6614fa..6c4d651d6a2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java @@ -33,6 +33,8 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; @@ -488,8 +490,24 @@ public void testPerUserResourcesXML() throws Exception { //Start RM so that it accepts app submissions rm.start(); try { - rm.submitApp(10, "app1", "user1", null, "b1"); - rm.submitApp(20, "app2", "user2", null, "b1"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(10, rm) + .withName("app1") + .withUser("user1") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + MockRMAppSubmitter.submit(rm, data1); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(20, rm) + .withName("app2") + .withUser("user2") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + MockRMAppSubmitter.submit(rm, data); //Get the XML from ws/v1/cluster/scheduler WebResource r = resource(); @@ -569,8 +587,24 @@ public void testPerUserResourcesJSON() throws Exception { //Start RM so that it accepts app submissions rm.start(); try { - rm.submitApp(10, "app1", "user1", null, "b1"); - rm.submitApp(20, "app2", "user2", null, "b1"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(10, rm) + .withName("app1") + .withUser("user1") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + MockRMAppSubmitter.submit(rm, data1); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(20, rm) + .withName("app2") + .withUser("user2") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + MockRMAppSubmitter.submit(rm, data); //Get JSON WebResource r = resource(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java index 40cf483cd3a..60f73a850bd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java @@ -32,6 +32,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; @@ -63,7 +65,15 @@ public void testAssignMultipleContainersPerNodeHeartbeat() nm.registerNode(); try { - RMApp app1 = rm.submitApp(10, "app1", "user1", null, "b1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(10, rm) + .withName("app1") + .withUser("user1") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm); am1.allocate(Arrays.asList(ResourceRequest .newInstance(Priority.UNDEFINED, "127.0.0.1", @@ -121,7 +131,15 @@ public void testAssignWithoutAvailableResource() throws Exception { nm.registerNode(); try { - RMApp app1 = rm.submitApp(1024, "app1", "user1", null, "b1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm) + .withName("app1") + .withUser("user1") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm); am1.allocate(Arrays.asList(ResourceRequest .newInstance(Priority.UNDEFINED, "127.0.0.1", @@ -204,7 +222,15 @@ public void testWrongNodeId() throws Exception { nm.registerNode(); try { - RMApp app1 = rm.submitApp(1024, "app1", "user1", null, "b1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm) + .withName("app1") + .withUser("user1") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm); am1.allocate(Arrays.asList(ResourceRequest .newInstance(Priority.UNDEFINED, "127.0.0.1", @@ -257,10 +283,26 @@ public void testReserveNewContainer() throws Exception { nm2.registerNode(); try { - RMApp app1 = rm.submitApp(10, "app1", "user1", null, "b1"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(10, rm) + .withName("app1") + .withUser("user1") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); - RMApp app2 = rm.submitApp(10, "app2", "user1", null, "b2"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(10, rm) + .withName("app2") + .withUser("user1") + .withAcls(null) + .withQueue("b2") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm2); am1.allocate(Arrays.asList(ResourceRequest @@ -375,7 +417,15 @@ public void testActivityJSON() throws Exception { nm.registerNode(); try { - RMApp app1 = rm.submitApp(10, "app1", "user1", null, "b1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(10, rm) + .withName("app1") + .withUser("user1") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); //Get JSON WebResource r = resource(); @@ -523,7 +573,15 @@ public void testAppActivityJSON() throws Exception { nm.registerNode(); try { - RMApp app1 = rm.submitApp(10, "app1", "user1", null, "b1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(10, rm) + .withName("app1") + .withUser("user1") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); //Get JSON WebResource r = resource(); @@ -569,7 +627,15 @@ public void testAppAssignMultipleContainersPerNodeHeartbeat() nm.registerNode(); try { - RMApp app1 = rm.submitApp(1024, "app1", "user1", null, "b1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm) + .withName("app1") + .withUser("user1") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm); am1.allocate(Arrays.asList(ResourceRequest .newInstance(Priority.UNDEFINED, "127.0.0.1", @@ -623,7 +689,15 @@ public void testAppAssignWithoutAvailableResource() throws Exception { nm.registerNode(); try { - RMApp app1 = rm.submitApp(1024, "app1", "user1", null, "b1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm) + .withName("app1") + .withUser("user1") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm); am1.allocate(Arrays.asList(ResourceRequest .newInstance(Priority.UNDEFINED, "127.0.0.1", @@ -667,7 +741,15 @@ public void testAppNoNM() throws Exception { rm.start(); try { - RMApp app1 = rm.submitApp(1024, "app1", "user1", null, "b1"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(1024, rm) + .withName("app1") + .withUser("user1") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); //Get JSON WebResource r = resource(); @@ -709,10 +791,26 @@ public void testAppReserveNewContainer() throws Exception { nm2.registerNode(); try { - RMApp app1 = rm.submitApp(10, "app1", "user1", null, "b1"); + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithDefaults(10, rm) + .withName("app1") + .withUser("user1") + .withAcls(null) + .withQueue("b1") + .withUnmanaged(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data1); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); - RMApp app2 = rm.submitApp(10, "app2", "user1", null, "b2"); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithDefaults(10, rm) + .withName("app2") + .withUser("user1") + .withAcls(null) + .withQueue("b2") + .withUnmanaged(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm, data); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm2); am1.allocate(Arrays.asList(ResourceRequest