diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java index ba43816de19..907147fa1be 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java @@ -37,6 +37,7 @@ import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.annotations.VisibleForTesting; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; import org.apache.commons.cli.GnuParser; @@ -74,9 +75,11 @@ @Private @Unstable public class SLSRunner { + // logger + public final static Logger LOG = Logger.getLogger(SLSRunner.class); + // RM, Runner private ResourceManager rm; - private static TaskRunner runner = new TaskRunner(); private String[] inputTraces; private Configuration conf; private Map queueAppNumMap; @@ -91,7 +94,6 @@ private Map amMap; private Set trackedApps; private Map amClassMap; - private static int remainingApps = 0; // metrics private String metricsOutputDir; @@ -100,19 +102,27 @@ // other simulation information private int numNMs, numRacks, numAMs, numTasks; private long maxRuntime; - public final static Map simulateInfoMap = - new HashMap(); - - // logger - public final static Logger LOG = Logger.getLogger(SLSRunner.class); // input traces, input-rumen or input-sls private boolean isSLS; + + // Static fields + // TODO, this is really a bad code style, need fix for better encapsulation + private static SLSRunner slsRunner; + private static TaskRunner taskRunner = new TaskRunner(); + private static int remainingApps = 0; + public static Map simulateInfoMap = new HashMap<>(); + public static boolean inUnitTest = false; public SLSRunner(boolean isSLS, String inputTraces[], String nodeFile, String outputDir, Set trackedApps, boolean printsimulation) throws IOException, ClassNotFoundException { + // Init static fields + SLSRunner.taskRunner = new TaskRunner(); + SLSRunner.simulateInfoMap = new HashMap<>(); + SLSRunner.remainingApps = 0; + this.isSLS = isSLS; this.inputTraces = inputTraces.clone(); this.nodeFile = nodeFile; @@ -125,13 +135,13 @@ public SLSRunner(boolean isSLS, String inputTraces[], String nodeFile, amMap = new ConcurrentHashMap<>(); amClassMap = new HashMap<>(); - // runner configuration + // taskRunner configuration conf = new Configuration(false); conf.addResource("sls-runner.xml"); - // runner + // taskRunner int poolSize = conf.getInt(SLSConfiguration.RUNNER_POOL_SIZE, SLSConfiguration.RUNNER_POOL_SIZE_DEFAULT); - SLSRunner.runner.setQueueSize(poolSize); + SLSRunner.taskRunner.setQueueSize(poolSize); // map for (Map.Entry e : conf) { String key = e.getKey().toString(); @@ -141,6 +151,11 @@ public SLSRunner(boolean isSLS, String inputTraces[], String nodeFile, } } } + + @VisibleForTesting + public Configuration getConfig() { + return conf; + } public void start() throws Exception { // start resource manager @@ -158,13 +173,28 @@ public void start() throws Exception { printSimulationInfo(); // blocked until all nodes RUNNING waitForNodesRunning(); - // starting the runner once everything is ready to go, - runner.start(); + // starting the taskRunner once everything is ready to go, + taskRunner.start(); + } + + public void stop() throws Exception { + rm.stop(); + taskRunner.stop(); } private void startRM() throws Exception { Configuration rmConf = new YarnConfiguration(); - String schedulerClass = rmConf.get(YarnConfiguration.RM_SCHEDULER); + + String schedulerClass; + + // Make unit test can easier set type of scheduler + // Look at conf first, if failed to find entry for rm_scheduler, load from + // rmConf + if (conf.get(YarnConfiguration.RM_SCHEDULER) != null) { + schedulerClass = conf.get(YarnConfiguration.RM_SCHEDULER); + } else { + schedulerClass = rmConf.get(YarnConfiguration.RM_SCHEDULER); + } // For CapacityScheduler we use a sub-classing instead of wrapping // to allow scheduler-specific invocations from monitors to work @@ -228,7 +258,7 @@ private void startNM() throws YarnException, IOException { nm.init(hostName, nmMemoryMB, nmVCores, random.nextInt(heartbeatInterval), heartbeatInterval, rm); nmMap.put(nm.getNode().getNodeID(), nm); - runner.schedule(nm); + taskRunner.schedule(nm); rackSet.add(nm.getNode().getRackName()); } numRacks = rackSet.size(); @@ -358,7 +388,7 @@ private void startAMFromSLSTraces(Resource containerResource, amSim.init(AM_ID++, heartbeatInterval, containerList, rm, this, jobStartTime, jobFinishTime, user, queue, isTracked, oldAppId); - runner.schedule(amSim); + taskRunner.schedule(amSim); maxRuntime = Math.max(maxRuntime, jobFinishTime); numTasks += containerList.size(); amMap.put(oldAppId, amSim); @@ -449,7 +479,7 @@ private void startAMFromRumenTraces(Resource containerResource, amSim.init(AM_ID ++, heartbeatInterval, containerList, rm, this, jobStartTimeMS, jobFinishTimeMS, user, jobQueue, isTracked, oldJobId); - runner.schedule(amSim); + taskRunner.schedule(amSim); maxRuntime = Math.max(maxRuntime, jobFinishTimeMS); numTasks += containerList.size(); amMap.put(oldJobId, amSim); @@ -510,8 +540,8 @@ private void printSimulationInfo() { return nmMap; } - public static TaskRunner getRunner() { - return runner; + public static TaskRunner getTaskRunner() { + return taskRunner; } public static void decreaseRemainingApps() { @@ -519,7 +549,11 @@ public static void decreaseRemainingApps() { if (remainingApps == 0) { LOG.info("SLSRunner tears down."); - System.exit(0); + + // Only call system.exit in a non unit test environment + if (!SLSRunner.inUnitTest) { + System.exit(0); + } } } @@ -571,8 +605,17 @@ public static void main(String args[]) throws Exception { boolean isSLS = inputSLS != null; String inputFiles[] = isSLS ? inputSLS.split(",") : inputRumen.split(","); - SLSRunner sls = new SLSRunner(isSLS, inputFiles, nodeFile, output, + slsRunner = new SLSRunner(isSLS, inputFiles, nodeFile, output, trackedJobSet, cmd.hasOption("printsimulation")); - sls.start(); + slsRunner.start(); + } + + @VisibleForTesting + public ResourceManager getRM() { + return rm; + } + + public static SLSRunner getSlsRunner() { + return slsRunner; } } diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java index a62f2b60240..0abbbfdf2dd 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java @@ -144,7 +144,7 @@ public void init(int id, int heartbeatInterval, @Override public void firstStep() throws Exception { simulateStartTimeMS = System.currentTimeMillis() - - SLSRunner.getRunner().getStartTimeMS(); + SLSRunner.getTaskRunner().getStartTimeMS(); // submit application, waiting until ACCEPTED submitApp(); @@ -218,7 +218,7 @@ public Object run() throws Exception { }); simulateFinishTimeMS = System.currentTimeMillis() - - SLSRunner.getRunner().getStartTimeMS(); + SLSRunner.getTaskRunner().getStartTimeMS(); // record job running information SchedulerMetrics schedulerMetrics = ((SchedulerWrapper)rm.getResourceScheduler()).getSchedulerMetrics(); diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/resourcemanager/MockAMLauncher.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/resourcemanager/MockAMLauncher.java index 20cf3e5fd91..baf975c863b 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/resourcemanager/MockAMLauncher.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/resourcemanager/MockAMLauncher.java @@ -65,6 +65,11 @@ protected void serviceStart() throws Exception { // Do nothing } + @Override + protected void serviceStop() { + // Do nothing + } + private void setupAMRMToken(RMAppAttempt appAttempt) { // Setup AMRMToken Token amrmToken = diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java index 7c37465d7b6..7e99608562c 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java @@ -110,6 +110,16 @@ public void setConf(Configuration conf) { } @Override + public void stop() { + try { + schedulerMetrics.stop(); + } catch (Exception e) { + e.printStackTrace(); + } + super.stop(); + } + + @Override public Allocation allocate(ApplicationAttemptId attemptId, List resourceRequests, List containerIds, List strings, List strings2, diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java index 572dacfc55d..6b42b767c42 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java @@ -104,6 +104,16 @@ public void setConf(Configuration conf) { } @Override + public void stop() { + try { + schedulerMetrics.stop(); + } catch (Exception e) { + e.printStackTrace(); + } + super.stop(); + } + + @Override public Allocation allocate(ApplicationAttemptId attemptId, List resourceRequests, List containerIds, List blacklistAdditions, List blacklistRemovals, diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java index a8792e81ec5..29b4509e16e 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java @@ -180,6 +180,10 @@ void init(ResourceScheduler resourceScheduler, Configuration config) jobRuntimeLogBW.flush(); } + public void stop() throws Exception { + web.stop(); + } + public MetricRegistry getMetrics() { return metrics; } diff --git a/hadoop-tools/hadoop-sls/src/test/data/sls-sample/slsjobs.json b/hadoop-tools/hadoop-sls/src/test/data/sls-sample/slsjobs.json new file mode 100644 index 00000000000..401e099f325 --- /dev/null +++ b/hadoop-tools/hadoop-sls/src/test/data/sls-sample/slsjobs.json @@ -0,0 +1,662 @@ +{ + "am.type": "mapreduce", + "job.start.ms": 0, + "job.end.ms": 100000000, + "job.queue.name": "sls_queue_1", + "job.user": "wangda", + "job.id": "job_0", + "job.am.resource.memory": "1024", + "job.tasks": [ + { + "container.host": "/default/192.168.0.3", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 1024, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.3", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 1024, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.5", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 1024, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.2", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 1024, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.4", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 1024, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.2", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 1024, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.5", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 1024, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.2", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 1024, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.1", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 1024, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.1", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 1024, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.2", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 1024, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.5", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 1024, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.1", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 1024, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.5", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 1024, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.1", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 1024, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.3", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 1024, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.3", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 1024, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.4", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 1024, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.3", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 1024, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.2", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 1024, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.5", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 1024, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.2", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 1024, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.4", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 1024, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.4", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 1024, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.5", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 1024, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.4", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 1024, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.2", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 1024, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.1", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 1024, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.2", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 1024, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.3", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 1024, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.2", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 1024, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.5", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 1024, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.5", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 1024, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.1", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 1024, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.2", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 1024, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.5", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 1024, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.3", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 1024, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.4", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 1024, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.5", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 1024, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.3", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 1024, + "container.end.ms": 1000 + } + ] +} +{ + "am.type": "mapreduce", + "job.start.ms": 10000, + "job.end.ms": 100000000, + "job.queue.name": "sls_queue_1", + "job.user": "wangda", + "job.id": "job_1", + "job.am.resource.memory": "2048", + "job.tasks": [ + { + "container.host": "/default/192.168.0.3", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 2048, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.1", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 2048, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.3", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 2048, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.4", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 2048, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.1", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 2048, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.4", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 2048, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.1", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 2048, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.5", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 2048, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.2", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 2048, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.1", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 2048, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.4", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 2048, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.5", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 2048, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.3", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 2048, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.2", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 2048, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.5", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 2048, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.4", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 2048, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.4", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 2048, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.1", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 2048, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.5", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 2048, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.5", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 2048, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.4", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 2048, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.4", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 2048, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.1", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 2048, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.1", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 2048, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.4", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 2048, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.2", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 2048, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.2", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 2048, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.2", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 2048, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.5", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 2048, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.3", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 2048, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.5", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 2048, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.3", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 2048, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.4", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 2048, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.2", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 2048, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.1", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 2048, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.5", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 2048, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.3", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 2048, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.1", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 2048, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.2", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 2048, + "container.end.ms": 1000 + }, + { + "container.host": "/default/192.168.0.3", + "container.priority": "20", + "container.type": "map", + "container.start.ms": 0, + "container.memory": 2048, + "container.end.ms": 1000 + } + ] +} diff --git a/hadoop-tools/hadoop-sls/src/test/data/sls-sample/slsnodes.json b/hadoop-tools/hadoop-sls/src/test/data/sls-sample/slsnodes.json new file mode 100644 index 00000000000..5a2e33d1835 --- /dev/null +++ b/hadoop-tools/hadoop-sls/src/test/data/sls-sample/slsnodes.json @@ -0,0 +1,20 @@ +{ + "rack": "default-rack", + "nodes": [ + { + "node": "192.168.0.0" + }, + { + "node": "192.168.0.1" + }, + { + "node": "192.168.0.2" + }, + { + "node": "192.168.0.3" + }, + { + "node": "192.168.0.4" + } + ] +} \ No newline at end of file diff --git a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/TestSLSRunner.java b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/TestSLSRunner.java index 9da8ef34a20..ad6eee660b0 100644 --- a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/TestSLSRunner.java +++ b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/TestSLSRunner.java @@ -18,7 +18,13 @@ package org.apache.hadoop.yarn.sls; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import java.io.File; @@ -26,12 +32,17 @@ import java.util.Collections; import java.util.List; import java.util.UUID; +import java.util.concurrent.ConcurrentMap; public class TestSLSRunner { + @Before + public void before() { + SLSRunner.inUnitTest = true; + } @Test @SuppressWarnings("all") - public void testSimulatorRunning() throws Exception { + public void testRumenSimulatorRunning() throws Exception { File tempDir = new File("target", UUID.randomUUID().toString()); final List exceptionList = Collections.synchronizedList(new ArrayList()); @@ -56,7 +67,7 @@ public void uncaughtException(Thread t, Throwable e) { Thread.sleep(1000); if (! exceptionList.isEmpty()) { - SLSRunner.getRunner().stop(); + SLSRunner.getTaskRunner().stop(); Assert.fail("TestSLSRunner catched exception from child thread " + "(TaskRunner.Task): " + exceptionList.get(0).getMessage()); break; @@ -64,7 +75,57 @@ public void uncaughtException(Thread t, Throwable e) { count--; } - SLSRunner.getRunner().stop(); + SLSRunner.getSlsRunner().stop(); + } + + private void verifySLSSimulatorRunningForScheduler(String scheduler) + throws Exception { + File tempDir = new File("target", UUID.randomUUID().toString()); + + // start the simulator + File slsOutputDir = new File(tempDir.getAbsolutePath() + "/slsoutput/"); + + SLSRunner sls = new SLSRunner(true, + new String[] { "src/test/data/sls-sample/slsjobs.json" }, + "src/test/data/sls-sample/slsnodes.json", + slsOutputDir.getAbsolutePath(), Collections.EMPTY_SET, true); + sls.getConfig().set("yarn.resourcemanager.scheduler.class", scheduler); + sls.start(); + + ResourceManager rm = sls.getRM(); + ConcurrentMap rmApps; + boolean anyAppIsRunning = false; + + while ((rmApps = rm.getRMContext().getRMApps()).size() < 2 + || anyAppIsRunning) { + anyAppIsRunning = false; + + // Make sure apps successfully finished + for (RMApp app : rmApps.values()) { + if (RMAppImpl.isAppInFinalState(app)) { + if (app.getState() != RMAppState.FINISHED) { + Assert.fail("App is not completed successfully"); + } + } else{ + anyAppIsRunning = true; + } + } + + Thread.sleep(200); + } + + sls.stop(); + } + + @Test(timeout = 120000) + public void testSLSSimulatorRunningForCapacityScheduler() throws Exception { + verifySLSSimulatorRunningForScheduler( + "org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler"); } + @Test(timeout = 120000) + public void testSLSSimulatorRunningForFairScheduler() throws Exception { + verifySLSSimulatorRunningForScheduler( + "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler"); + } } diff --git a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/appmaster/TestAMSimulator.java b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/appmaster/TestAMSimulator.java index ca3d1958a3b..b7c587cea1b 100644 --- a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/appmaster/TestAMSimulator.java +++ b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/appmaster/TestAMSimulator.java @@ -24,6 +24,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; +import org.apache.hadoop.yarn.sls.SLSRunner; import org.apache.hadoop.yarn.sls.conf.SLSConfiguration; import org.apache.hadoop.yarn.sls.scheduler.*; import org.junit.After; diff --git a/hadoop-tools/hadoop-sls/src/test/resources/capacity-scheduler.xml b/hadoop-tools/hadoop-sls/src/test/resources/capacity-scheduler.xml index 61be96ae6d4..669f723cd35 100644 --- a/hadoop-tools/hadoop-sls/src/test/resources/capacity-scheduler.xml +++ b/hadoop-tools/hadoop-sls/src/test/resources/capacity-scheduler.xml @@ -34,6 +34,11 @@ + yarn.scheduler.capacity.root.sls_queue_1.user-limit-factor + 10 + + + yarn.scheduler.capacity.root.sls_queue_1.maximum-capacity 100 diff --git a/hadoop-tools/hadoop-sls/src/test/resources/log4j.properties b/hadoop-tools/hadoop-sls/src/test/resources/log4j.properties new file mode 100644 index 00000000000..9b581db9061 --- /dev/null +++ b/hadoop-tools/hadoop-sls/src/test/resources/log4j.properties @@ -0,0 +1,19 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# log4j configuration used during build and unit tests + +log4j.rootLogger=info,stdout +log4j.threshold=ALL +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2} (%F:%M(%L)) - %m%n \ No newline at end of file diff --git a/hadoop-tools/hadoop-sls/src/test/resources/yarn-site.xml b/hadoop-tools/hadoop-sls/src/test/resources/yarn-site.xml index 78aa6f2dd7a..06394bacde2 100644 --- a/hadoop-tools/hadoop-sls/src/test/resources/yarn-site.xml +++ b/hadoop-tools/hadoop-sls/src/test/resources/yarn-site.xml @@ -79,4 +79,10 @@ yarn.scheduler.fair.assignmultiple true + + + Set to false, to avoid ip check + hadoop.security.token.service.use_ip + true +