diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncSchedulingChaosMonkey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncSchedulingChaosMonkey.java new file mode 100644 index 00000000000..56512902010 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncSchedulingChaosMonkey.java @@ -0,0 +1,147 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; + +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.monitor.SchedulingEditPolicy; +import org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.chaosmonkey.ChaosMonkeyPlayground; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.chaosmonkey.rm.monkeys.AddAppMonkey; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.chaosmonkey.rm.monkeys.AddNodeMonkey; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.chaosmonkey.rm.monkeys.AppCleanerMonkey; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.chaosmonkey.rm.monkeys.NMHeartbeatMonkey; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.chaosmonkey.rm.monkeys.RMMonkeyContexts; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.chaosmonkey.rm.monkeys.RemoveAppMonkey; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.chaosmonkey.rm.monkeys.RemoveNodeMonkey; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.chaosmonkey.rm.monkeys.UpdateAppResourceMonkey; +import org.apache.log4j.Level; +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; +import org.junit.Assume; +import org.junit.Test; + +import java.util.concurrent.TimeUnit; + +public class TestCapacitySchedulerAsyncSchedulingChaosMonkey { + private final int GB = 1024; + + private YarnConfiguration conf; + + RMNodeLabelsManager mgr; + + @Test + public void testChaosMonkeyAsyncCSOneThread() throws Exception { + testChaosMonkeyForAsyncCS(1); + } + + private void testChaosMonkeyForAsyncCS(int numThreads) throws Exception { + conf = new YarnConfiguration(); + + // Enable scheduler and async scheduling + conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, + ResourceScheduler.class); + conf.setBoolean( + CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_ENABLE, true); + mgr = new NullRMNodeLabelsManager(); + mgr.init(conf); + + // Since this is more of a performance unit test, only run if + // RunUserLimitThroughput is set (-DRunUserLimitThroughput=true) + Assume.assumeTrue(Boolean.valueOf( + System.getProperty("RunCapacitySchedulerChaosMonkeyTests"))); + + conf.setInt( + CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_MAXIMUM_THREAD, + numThreads); + conf.setInt(CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_PREFIX + + ".scheduling-interval-ms", 10); + + // Enable preemption policy + conf.setBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, true); + conf.setClass(YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES, + ProportionalCapacityPreemptionPolicy.class, SchedulingEditPolicy.class); + conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 100 * GB); + + // Set preemption related configurations + conf.setInt(CapacitySchedulerConfiguration.PREEMPTION_WAIT_TIME_BEFORE_KILL, + 0); + conf.setFloat(CapacitySchedulerConfiguration.TOTAL_PREEMPTION_PER_ROUND, + 1.0f); + conf.setFloat( + CapacitySchedulerConfiguration.PREEMPTION_NATURAL_TERMINATION_FACTOR, + 1.0f); + conf.setLong(CapacitySchedulerConfiguration.PREEMPTION_MONITORING_INTERVAL, + 1000L); + + final RMNodeLabelsManager mgr = new NullRMNodeLabelsManager(); + mgr.init(conf); + + // inject node label manager + MockRM rm = new MockRM(TestUtils.getConfigurationWithMultipleQueues(conf)) { + @Override + public RMNodeLabelsManager createNodeLabelManager() { + return mgr; + } + }; + + rm.getRMContext().setNodeLabelManager(mgr); + rm.start(); + + Logger rootLogger = LogManager.getRootLogger(); + rootLogger.setLevel(Level.INFO); + + ChaosMonkeyPlayground cmp = new ChaosMonkeyPlayground(50, 60 * 1000); + RMMonkeyContexts rmMonkeyContexts = new RMMonkeyContexts(rm); + + cmp.addBlockingMonkeyToArmy(new AddAppMonkey(rmMonkeyContexts, + TimeUnit.NANOSECONDS.convert(10, TimeUnit.MILLISECONDS), 0.9f)); + + cmp.addMonkeyToArmy(new RemoveAppMonkey(rmMonkeyContexts, + TimeUnit.NANOSECONDS.convert(50, TimeUnit.MILLISECONDS), 0.8f)); + + cmp.addMonkeyToArmy(new AddNodeMonkey(rmMonkeyContexts, + TimeUnit.NANOSECONDS.convert(10, TimeUnit.MILLISECONDS), 0.9f)); + + cmp.addMonkeyToArmy(new RemoveNodeMonkey(rmMonkeyContexts, + TimeUnit.NANOSECONDS.convert(50, TimeUnit.MILLISECONDS), 0.1f)); + + cmp.addMonkeyToArmy(new UpdateAppResourceMonkey(rmMonkeyContexts, + TimeUnit.NANOSECONDS.convert(10, TimeUnit.MILLISECONDS), 0.1f)); + + cmp.addMonkeyToArmy(new NMHeartbeatMonkey(rmMonkeyContexts, + TimeUnit.NANOSECONDS.convert(1, TimeUnit.MILLISECONDS))); + + cmp.addCleanerMonkeyToArmy(new AppCleanerMonkey(rmMonkeyContexts)); + + cmp.startDoMonkeyThings(); + + cmp.waitAndCleanupMonkeyThings(); + + // Check it. + QueueMetrics rootQueueMetrics = rm.getResourceScheduler().getRootQueueMetrics(); + System.out.println("Total allocated containers = " + rootQueueMetrics + .getAggregateAllocatedContainers()); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/chaosmonkey/ChaosMonkeyPlayground.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/chaosmonkey/ChaosMonkeyPlayground.java new file mode 100644 index 00000000000..a7f220eedbd --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/chaosmonkey/ChaosMonkeyPlayground.java @@ -0,0 +1,92 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.chaosmonkey; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +public class ChaosMonkeyPlayground { + protected List monkeyArmy = new ArrayList<>(); + protected List blockingMonkeyArmy = new ArrayList<>(); + protected List cleannerMonkeyArmy = new ArrayList<>(); + + private ScheduledThreadPoolExecutor executorService; + private ScheduledThreadPoolExecutor blockingExecutorServices; + + public ChaosMonkeyPlayground(int threads, int runTimeMs) { + executorService = new ScheduledThreadPoolExecutor(threads); + blockingExecutorServices = new ScheduledThreadPoolExecutor(threads); + + // Stop the executor service in x time. + new java.util.Timer().schedule(new java.util.TimerTask() { + @Override + public void run() { + executorService.shutdownNow(); + blockingExecutorServices.shutdownNow(); + } + }, runTimeMs); + } + + public void addMonkeyToArmy(Monkey monkey) { + monkeyArmy.add(monkey); + } + + public void addBlockingMonkeyToArmy(Monkey monkey) { + blockingMonkeyArmy.add(monkey); + } + + public void addCleanerMonkeyToArmy(Monkey monkey) { + cleannerMonkeyArmy.add(monkey); + } + + public void startDoMonkeyThings() { + for (Monkey monkey : monkeyArmy) { + System.out.println( + "Adding monkey=" + monkey.getClass().getName() + " interval=" + monkey + .getNanoRunningInterval() + " ns."); + executorService.scheduleAtFixedRate(monkey, 0, + monkey.getNanoRunningInterval(), TimeUnit.NANOSECONDS); + } + + for (Monkey monkey : blockingMonkeyArmy) { + System.out.println( + "Adding monkey=" + monkey.getClass().getName() + " interval=" + monkey + .getNanoRunningInterval() + " ns."); + blockingExecutorServices.scheduleAtFixedRate(monkey, 0, + monkey.getNanoRunningInterval(), TimeUnit.NANOSECONDS); + } + } + + public void waitAndCleanupMonkeyThings() throws InterruptedException { + while (true) { + if (!executorService.isTerminated() || !blockingExecutorServices + .isTerminated()) { + Thread.sleep(1); + continue; + } + + for (Monkey monkey : cleannerMonkeyArmy) { + monkey.run(); + } + break; + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/chaosmonkey/Monkey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/chaosmonkey/Monkey.java new file mode 100644 index 00000000000..46eed8b5d7e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/chaosmonkey/Monkey.java @@ -0,0 +1,27 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.chaosmonkey; + +public abstract class Monkey implements Runnable { + protected long nanoInterval; + + public long getNanoRunningInterval() { + return nanoInterval; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/chaosmonkey/ProbBasedMonkey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/chaosmonkey/ProbBasedMonkey.java new file mode 100644 index 00000000000..db61fefdbaf --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/chaosmonkey/ProbBasedMonkey.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.chaosmonkey; + +import java.util.Random; + +public abstract class ProbBasedMonkey extends Monkey { + public ProbBasedMonkey(long nanoInterval, float prob) { + super.nanoInterval = nanoInterval; + this.prob = prob; + } + + protected Random random = new Random(); + + protected float prob; + + public float getProb() { + return prob; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/chaosmonkey/rm/monkeys/AddAppMonkey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/chaosmonkey/rm/monkeys/AddAppMonkey.java new file mode 100644 index 00000000000..29261e1f931 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/chaosmonkey/rm/monkeys/AddAppMonkey.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.chaosmonkey.rm.monkeys; + +import org.apache.hadoop.yarn.server.resourcemanager.MockAM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; + +public class AddAppMonkey extends RMBaseMonkey { + public AddAppMonkey(RMMonkeyContexts monkeyContexts, long nanoInterval, + float prob) { + super(monkeyContexts, nanoInterval, prob); + } + + @Override + public void run() { + if (random.nextDouble() > getProb()) { + return; + } + + try { + if (monkeyContexts.getApps().size() >= monkeyContexts.getMaximumApps()) { + return; + } + + MockRM rm = monkeyContexts.getRm(); + RMApp rmApp = monkeyContexts.getRm().submitApp(getRandomAsk().getMemory(), + "app", "user", null, false, monkeyContexts.getLeafQueues() + .get(random.nextInt(monkeyContexts.getLeafQueues().size())), 1, + null, null, false); + MockAM am = MockRM.launchAMWhenAsyncSchedulingEnabled(rmApp, rm); + am.registerAppAttempt(false); + monkeyContexts.getApps().put(am.getApplicationAttemptId(), am); + } catch (Exception e) { + e.printStackTrace(); + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/chaosmonkey/rm/monkeys/AddNodeMonkey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/chaosmonkey/rm/monkeys/AddNodeMonkey.java new file mode 100644 index 00000000000..d9f9fdef6ed --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/chaosmonkey/rm/monkeys/AddNodeMonkey.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.chaosmonkey.rm.monkeys; + +import org.apache.hadoop.yarn.server.resourcemanager.MockNM; + +public class AddNodeMonkey extends RMBaseMonkey { + public AddNodeMonkey(RMMonkeyContexts monkeyContexts, long nanoInterval, + float prob) { + super(monkeyContexts, nanoInterval, prob); + } + + @Override + public void run() { + if (random.nextDouble() > getProb()) { + return; + } + + try { + if (monkeyContexts.getNodes().size() > monkeyContexts.getMaximumNodes()) { + return; + } + + MockNM nm = monkeyContexts.getRm().registerNode( + "127." + random.nextInt(200) + "." + random.nextInt(200) + "." + + random.nextInt(200) + ":1234", + getMaximumAsk().getMemory() * 2); + monkeyContexts.getNodes().put(nm.getNodeId(), nm); + } catch (Exception e) { + e.printStackTrace(); + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/chaosmonkey/rm/monkeys/AppCleanerMonkey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/chaosmonkey/rm/monkeys/AppCleanerMonkey.java new file mode 100644 index 00000000000..e2eba7d2065 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/chaosmonkey/rm/monkeys/AppCleanerMonkey.java @@ -0,0 +1,22 @@ +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.chaosmonkey.rm.monkeys; + +import org.apache.hadoop.yarn.server.resourcemanager.MockAM; + +public class AppCleanerMonkey extends RMBaseMonkey { + public AppCleanerMonkey(RMMonkeyContexts monkeyContexts) { + super(monkeyContexts, -1, 1.0f); + } + + @Override + public void run() { + while (monkeyContexts.getApps().size() > 0) { + for (MockAM am : monkeyContexts.getApps().values()) { + try { + am.unregisterAppAttempt(false); + } catch (Exception e) { + e.printStackTrace(); + } + } + } + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/chaosmonkey/rm/monkeys/NMHeartbeatMonkey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/chaosmonkey/rm/monkeys/NMHeartbeatMonkey.java new file mode 100644 index 00000000000..ba7bd917d97 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/chaosmonkey/rm/monkeys/NMHeartbeatMonkey.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.chaosmonkey.rm.monkeys; + +import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.server.resourcemanager.MockNM; + +public class NMHeartbeatMonkey extends RMBaseMonkey { + public NMHeartbeatMonkey(RMMonkeyContexts monkeyContexts, long nanoInterval) { + super(monkeyContexts, nanoInterval, 1.0f); + } + + @Override + public void run() { + try { + NodeId nodeId = (NodeId) randomPick(monkeyContexts.getNodes().keySet()); + if (nodeId == null) { + return; + } + + MockNM nm = monkeyContexts.getNodes().remove(nodeId); + if (nm != null) { + // 1% chance to report unhealthy + nm.nodeHeartbeat(random.nextDouble() < 0.99 ? true : false); + } + } catch (Exception e) { + e.printStackTrace(); + } + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/chaosmonkey/rm/monkeys/RMBaseMonkey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/chaosmonkey/rm/monkeys/RMBaseMonkey.java new file mode 100644 index 00000000000..33d3e4788bc --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/chaosmonkey/rm/monkeys/RMBaseMonkey.java @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.chaosmonkey.rm.monkeys; + +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.chaosmonkey.ProbBasedMonkey; +import org.apache.hadoop.yarn.util.resource.Resources; + +import java.util.Collection; + +public abstract class RMBaseMonkey extends ProbBasedMonkey { + RMMonkeyContexts monkeyContexts; + + public RMBaseMonkey(RMMonkeyContexts monkeyContexts, long nanoInterval, + float prob) { + super(nanoInterval, prob); + this.monkeyContexts = monkeyContexts; + } + + Object randomPick(Collection c) { + int num = (int) (Math.random() * c.size()); + for (Object t : c) + if (--num < 0) + return t; + return null; + } + + Resource getMaximumAsk() { + Resource maximum = monkeyContexts.getRm().getResourceScheduler() + .getMaximumResourceCapability(); + return maximum; + } + + Resource getRandomAsk() { + Resource maximum = monkeyContexts.getRm().getResourceScheduler() + .getMaximumResourceCapability(); + Resource ask = Resources.multiply(maximum, random.nextDouble()); + return ask; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/chaosmonkey/rm/monkeys/RMMonkeyContexts.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/chaosmonkey/rm/monkeys/RMMonkeyContexts.java new file mode 100644 index 00000000000..da722c430a5 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/chaosmonkey/rm/monkeys/RMMonkeyContexts.java @@ -0,0 +1,94 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.chaosmonkey.rm.monkeys; + +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.QueueInfo; +import org.apache.hadoop.yarn.server.resourcemanager.MockAM; +import org.apache.hadoop.yarn.server.resourcemanager.MockNM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRM; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +public class RMMonkeyContexts { + private Map apps = new ConcurrentHashMap<>(); + private Map nodes = new ConcurrentHashMap<>(); + private MockRM rm; + + private int maximumApps = 5000; + private int maximumNodes = 10000; + + private List leafQueues = new ArrayList<>(); + + public RMMonkeyContexts(MockRM rm) throws IOException { + this.rm = rm; + QueueInfo queueInfo = rm.getResourceScheduler().getQueueInfo("root", true, + true); + getAllChildQueue(queueInfo); + } + + private void getAllChildQueue(QueueInfo queue) { + if (queue.getChildQueues() == null || queue.getChildQueues().isEmpty()) { + leafQueues.add(queue.getQueueName()); + return; + } + + for (QueueInfo child : queue.getChildQueues()) { + getAllChildQueue(child); + } + } + + public Map getApps() { + return apps; + } + + public Map getNodes() { + return nodes; + } + + public MockRM getRm() { + return rm; + } + + public List getLeafQueues() { + return leafQueues; + } + + + public int getMaximumApps() { + return maximumApps; + } + + public void setMaximumApps(int maximumApps) { + this.maximumApps = maximumApps; + } + + public int getMaximumNodes() { + return maximumNodes; + } + + public void setMaximumNodes(int maximumNodes) { + this.maximumNodes = maximumNodes; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/chaosmonkey/rm/monkeys/RemoveAppMonkey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/chaosmonkey/rm/monkeys/RemoveAppMonkey.java new file mode 100644 index 00000000000..7d1c777b272 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/chaosmonkey/rm/monkeys/RemoveAppMonkey.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.chaosmonkey.rm.monkeys; + +import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptRequest; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.server.resourcemanager.MockAM; + +public class RemoveAppMonkey extends RMBaseMonkey { + public RemoveAppMonkey(RMMonkeyContexts monkeyContexts, long nanoInterval, + float prob) { + super(monkeyContexts, nanoInterval, prob); + } + + @Override + public void run() { + if (random.nextDouble() > getProb()) { + return; + } + + try { + ApplicationAttemptId attemptId = (ApplicationAttemptId) randomPick( + monkeyContexts.getApps().keySet()); + if (attemptId == null) { + return; + } + + MockAM am = monkeyContexts.getApps().remove(attemptId); + + monkeyContexts.getRm().getClientRMService().failApplicationAttempt( + FailApplicationAttemptRequest + .newInstance(am.getApplicationAttemptId())); + } catch (Exception e) { + e.printStackTrace(); + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/chaosmonkey/rm/monkeys/RemoveNodeMonkey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/chaosmonkey/rm/monkeys/RemoveNodeMonkey.java new file mode 100644 index 00000000000..bccd33148d4 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/chaosmonkey/rm/monkeys/RemoveNodeMonkey.java @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.chaosmonkey.rm.monkeys; + +import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.server.resourcemanager.MockNM; + +public class RemoveNodeMonkey extends RMBaseMonkey { + public RemoveNodeMonkey(RMMonkeyContexts monkeyContexts, long nanoInterval, + float prob) { + super(monkeyContexts, nanoInterval, prob); + } + + @Override + public void run() { + if (random.nextDouble() > getProb()) { + return; + } + + try { + NodeId nodeId = (NodeId) randomPick(monkeyContexts.getNodes().keySet()); + if (nodeId == null) { + return; + } + + MockNM nm = monkeyContexts.getNodes().remove(nodeId); + if (nm != null) { + nm.unRegisterNode(); + } + } catch (Exception e) { + e.printStackTrace(); + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/chaosmonkey/rm/monkeys/UpdateAppResourceMonkey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/chaosmonkey/rm/monkeys/UpdateAppResourceMonkey.java new file mode 100644 index 00000000000..ed1e3c29a5a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/chaosmonkey/rm/monkeys/UpdateAppResourceMonkey.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.chaosmonkey.rm.monkeys; + +import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; +import org.apache.hadoop.yarn.server.resourcemanager.MockAM; + +public class UpdateAppResourceMonkey extends RMBaseMonkey { + public UpdateAppResourceMonkey(RMMonkeyContexts monkeyContexts, + long nanoInterval, float prob) { + super(monkeyContexts, nanoInterval, prob); + } + + @Override + public void run() { + try { + MockAM am = (MockAM) randomPick(monkeyContexts.getApps().values()); + if (am == null) { + return; + } + + am.allocate("*", getRandomAsk().getMemory(), random.nextInt(100), + random.nextInt(5), null, CommonNodeLabelsManager.NO_LABEL); + } catch (Exception e) { + e.printStackTrace(); + } + } +}