From 7a8ab37b5f3f264b3e569d2844f4bd958deb2277 Mon Sep 17 00:00:00 2001 From: Prabhu Joseph Date: Fri, 1 Feb 2019 15:46:02 +0000 Subject: [PATCH] YARN-9253 --- .../distributedshell/TestDistributedShell.java | 77 +++++++++++++++++++++- 1 file changed, 74 insertions(+), 3 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java index e67e541..7c0cc5d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java @@ -107,6 +107,7 @@ import org.junit.rules.Timeout; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.junit.rules.TestName; public class TestDistributedShell { @@ -120,7 +121,7 @@ protected YarnConfiguration conf = null; // location of the filesystem timeline writer for timeline service v.2 private String timelineV2StorageDir = null; - private static final int NUM_NMS = 1; + private static int numNms = 1; private static final float DEFAULT_TIMELINE_VERSION = 1.0f; private static final String TIMELINE_AUX_SERVICE_NAME = "timeline_collector"; private static final int MIN_ALLOCATION_MB = 128; @@ -135,10 +136,18 @@ public Timeout globalTimeout = new Timeout(90000); @Rule public TemporaryFolder tmpFolder = new TemporaryFolder(); + @Rule + public TestName testName = new TestName(); @Before public void setup() throws Exception { - setupInternal(NUM_NMS, timelineVersionWatcher.getTimelineVersion()); + if (testName.getMethodName().equals( + "testDistributedShellWithPlacementConstraint")) { + numNms = 2; + } else { + numNms = 1; + } + setupInternal(numNms, timelineVersionWatcher.getTimelineVersion()); } protected void setupInternal(int numNodeManager) throws Exception { @@ -1283,7 +1292,7 @@ protected void waitForNMsToRegister() throws Exception { int sec = 60; while (sec >= 0) { if (yarnCluster.getResourceManager().getRMContext().getRMNodes().size() - >= NUM_NMS) { + >= numNms) { break; } Thread.sleep(1000); @@ -1790,4 +1799,66 @@ public void testDistributedShellWithNonExistentFileLocalization() client.init(args); client.run(); } + + @TimelineVersion(2.0f) + @Test + public void testDistributedShellWithPlacementConstraint() + throws Exception { + String[] args = { + "--jar", + APPMASTER_JAR, + "1", + "--shell_command", + getSleepCommand(60), + "--placement_spec", + "zk=1,NOTIN,NODE,zk:spark=1,NOTIN,NODE,zk" + }; + + final AtomicBoolean result = new AtomicBoolean(false); + Client client = new Client(new Configuration(yarnCluster.getConfig())); + client.init(args); + Thread t = new Thread() { + public void run() { + try { + result.set(client.run()); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + }; + t.start(); + + YarnClient yarnClient = YarnClient.createYarnClient(); + yarnClient.init(new Configuration(yarnCluster.getConfig())); + yarnClient.start(); + + while (true) { + List apps = yarnClient.getApplications(); + if (apps.isEmpty()) { + Thread.sleep(10); + continue; + } + ApplicationReport appReport = apps.get(0); + ApplicationId appId = appReport.getApplicationId(); + List appAttempts = + yarnClient.getApplicationAttempts(appId); + if (appAttempts.isEmpty()) { + Thread.sleep(10); + continue; + } + ApplicationAttemptReport appAttemptReport = appAttempts.get(0); + List containers = yarnClient. + getContainers(appAttemptReport.getApplicationAttemptId()); + if (containers.size() < 2) { + Thread.sleep(10); + continue; + } + Assert.assertNotEquals(containers.get(0).getAssignedNode(), + containers.get(1).getAssignedNode()); + yarnClient.killApplication(appId); + return; + } + } + } + -- 1.8.3.1