From 0a204ee1368eb6fcf4248226b43dcafe14d00355 Mon Sep 17 00:00:00 2001 From: Elliott Clark Date: Thu, 29 Aug 2013 14:05:22 -0700 Subject: [PATCH] more monkies More logging --- .../hadoop/hbase/client/RpcRetryingCaller.java | 2 +- .../hbase/chaos/actions/BatchRestartRsAction.java | 3 +- .../actions/CompactRandomRegionOfTableAction.java | 4 +- .../actions/FlushRandomRegionOfTableAction.java | 4 +- .../actions/MoveRandomRegionOfTableAction.java | 4 +- .../hbase/chaos/actions/RestartRandomRsAction.java | 4 +- .../chaos/actions/RollingBatchRestartRsAction.java | 4 +- .../actions/SplitRandomRegionOfTableAction.java | 4 +- .../hbase/chaos/factories/MonkeyFactory.java | 25 ++-- .../NoMergeSlowDeterministicMonkeyFactory.java | 81 +++++++++++++ .../NoMoveSlowDeterministicMonkeyFactory.java | 79 +++++++++++++ .../chaos/factories/RestartMonkeyFactory.java | 44 +++++++ .../chaos/factories/VerySlowMonkeyFactory.java | 63 ++++++++++ .../hadoop/hbase/chaos/monkies/ChaosMonkey.java | 50 ++++++++ .../chaos/monkies/PolicyBasedChaosMonkey.java | 49 -------- .../chaos/policies/PeriodicRandomActionPolicy.java | 4 +- .../hbase/test/IntegrationTestBigLinkedList.java | 5 +- .../hadoop/hbase/mapreduce/HLogInputFormat.java | 2 +- .../hbase/regionserver/wal/HLogPrettyPrinter.java | 3 +- .../apache/hadoop/hbase/util/RowLogSearchJob.java | 120 ++++++++++++++++++++ 20 files changed, 478 insertions(+), 76 deletions(-) create mode 100644 hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/NoMergeSlowDeterministicMonkeyFactory.java create mode 100644 hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/NoMoveSlowDeterministicMonkeyFactory.java create mode 100644 hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/RestartMonkeyFactory.java create mode 100644 hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/VerySlowMonkeyFactory.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/util/RowLogSearchJob.java diff --git hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCaller.java hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCaller.java index e501d70..ea4ef89 100644 --- hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCaller.java +++ hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCaller.java @@ -166,7 +166,7 @@ public class RpcRetryingCaller { /** * Call the server once only. - * {@link RetryingCallable} has a strange shape so we can do retrys. Use this invocation if you + * {@link RetryingCallable} has a strange shape so we can do retries. Use this invocation if you * want to do a single call only (A call to {@link RetryingCallable#call()} will not likely * succeed). * @return an object of type T diff --git hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/BatchRestartRsAction.java hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/BatchRestartRsAction.java index edfd9c4..b425197 100644 --- hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/BatchRestartRsAction.java +++ hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/BatchRestartRsAction.java @@ -21,6 +21,7 @@ package org.apache.hadoop.hbase.chaos.actions; import java.util.List; import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.chaos.monkies.ChaosMonkey; import org.apache.hadoop.hbase.chaos.monkies.PolicyBasedChaosMonkey; /** @@ -38,7 +39,7 @@ public class BatchRestartRsAction extends RestartActionBaseAction { public void perform() throws Exception { LOG.info(String.format("Performing action: Batch restarting %d%% of region servers", (int)(ratio * 100))); - List selectedServers = PolicyBasedChaosMonkey.selectRandomItems(getCurrentServers(), + List selectedServers = ChaosMonkey.selectRandomItems(getCurrentServers(), ratio); for (ServerName server : selectedServers) { diff --git hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/CompactRandomRegionOfTableAction.java hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/CompactRandomRegionOfTableAction.java index cba06dc..2b17df0 100644 --- hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/CompactRandomRegionOfTableAction.java +++ hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/CompactRandomRegionOfTableAction.java @@ -23,7 +23,7 @@ import java.util.List; import org.apache.commons.lang.math.RandomUtils; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.chaos.monkies.PolicyBasedChaosMonkey; +import org.apache.hadoop.hbase.chaos.monkies.ChaosMonkey; import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.util.Bytes; @@ -58,7 +58,7 @@ public class CompactRandomRegionOfTableAction extends Action { LOG.info("Performing action: Compact random region of table " + tableName + ", major=" + major); - HRegionInfo region = PolicyBasedChaosMonkey.selectRandomItem( + HRegionInfo region = ChaosMonkey.selectRandomItem( regions.toArray(new HRegionInfo[regions.size()])); if (major) { diff --git hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/FlushRandomRegionOfTableAction.java hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/FlushRandomRegionOfTableAction.java index c4ac4d8..154f86a 100644 --- hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/FlushRandomRegionOfTableAction.java +++ hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/FlushRandomRegionOfTableAction.java @@ -22,7 +22,7 @@ import java.util.List; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.chaos.monkies.PolicyBasedChaosMonkey; +import org.apache.hadoop.hbase.chaos.monkies.ChaosMonkey; import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.util.Bytes; @@ -51,7 +51,7 @@ public class FlushRandomRegionOfTableAction extends Action { LOG.info("Performing action: Flush random region of table " + tableName); List regions = admin.getTableRegions(tableNameBytes); - HRegionInfo region = PolicyBasedChaosMonkey.selectRandomItem( + HRegionInfo region = ChaosMonkey.selectRandomItem( regions.toArray(new HRegionInfo[regions.size()])); LOG.debug("Flushing region " + region.getRegionNameAsString()); admin.flush(region.getRegionName()); diff --git hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/MoveRandomRegionOfTableAction.java hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/MoveRandomRegionOfTableAction.java index 9652a5a..c007eac 100644 --- hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/MoveRandomRegionOfTableAction.java +++ hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/MoveRandomRegionOfTableAction.java @@ -22,7 +22,7 @@ import java.util.List; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.chaos.monkies.PolicyBasedChaosMonkey; +import org.apache.hadoop.hbase.chaos.monkies.ChaosMonkey; import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.util.Bytes; @@ -51,7 +51,7 @@ public class MoveRandomRegionOfTableAction extends Action { LOG.info("Performing action: Move random region of table " + tableName); List regions = admin.getTableRegions(tableNameBytes); - HRegionInfo region = PolicyBasedChaosMonkey.selectRandomItem( + HRegionInfo region = ChaosMonkey.selectRandomItem( regions.toArray(new HRegionInfo[regions.size()])); LOG.debug("Unassigning region " + region.getRegionNameAsString()); admin.unassign(region.getRegionName(), false); diff --git hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RestartRandomRsAction.java hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RestartRandomRsAction.java index 7b09dd3..acff912 100644 --- hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RestartRandomRsAction.java +++ hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RestartRandomRsAction.java @@ -19,7 +19,7 @@ package org.apache.hadoop.hbase.chaos.actions; import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.chaos.monkies.PolicyBasedChaosMonkey; +import org.apache.hadoop.hbase.chaos.monkies.ChaosMonkey; /** * Action that restarts a random HRegionServer @@ -32,7 +32,7 @@ public class RestartRandomRsAction extends RestartActionBaseAction { @Override public void perform() throws Exception { LOG.info("Performing action: Restart random region server"); - ServerName server = PolicyBasedChaosMonkey.selectRandomItem(getCurrentServers()); + ServerName server = ChaosMonkey.selectRandomItem(getCurrentServers()); restartRs(server, sleepTime); } diff --git hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RollingBatchRestartRsAction.java hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RollingBatchRestartRsAction.java index 4b6ccfb..c8532ba 100644 --- hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RollingBatchRestartRsAction.java +++ hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RollingBatchRestartRsAction.java @@ -24,7 +24,7 @@ import java.util.Queue; import org.apache.commons.lang.math.RandomUtils; import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.chaos.monkies.PolicyBasedChaosMonkey; +import org.apache.hadoop.hbase.chaos.monkies.ChaosMonkey; /** * Restarts a ratio of the regionservers in a rolling fashion. At each step, either kills a @@ -39,7 +39,7 @@ public class RollingBatchRestartRsAction extends BatchRestartRsAction { public void perform() throws Exception { LOG.info(String.format("Performing action: Rolling batch restarting %d%% of region servers", (int)(ratio * 100))); - List selectedServers = PolicyBasedChaosMonkey.selectRandomItems(getCurrentServers(), + List selectedServers = ChaosMonkey.selectRandomItems(getCurrentServers(), ratio); Queue serversToBeKilled = new LinkedList(selectedServers); diff --git hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/SplitRandomRegionOfTableAction.java hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/SplitRandomRegionOfTableAction.java index 116a447..c7f87ea 100644 --- hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/SplitRandomRegionOfTableAction.java +++ hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/SplitRandomRegionOfTableAction.java @@ -22,7 +22,7 @@ import java.util.List; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.chaos.monkies.PolicyBasedChaosMonkey; +import org.apache.hadoop.hbase.chaos.monkies.ChaosMonkey; import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.util.Bytes; @@ -51,7 +51,7 @@ public class SplitRandomRegionOfTableAction extends Action { LOG.info("Performing action: Split random region of table " + tableName); List regions = admin.getTableRegions(tableNameBytes); - HRegionInfo region = PolicyBasedChaosMonkey.selectRandomItem( + HRegionInfo region = ChaosMonkey.selectRandomItem( regions.toArray(new HRegionInfo[regions.size()])); LOG.debug("Splitting region " + region.getRegionNameAsString()); admin.split(region.getRegionName()); diff --git hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/MonkeyFactory.java hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/MonkeyFactory.java index 4488b86..4cd97db 100644 --- hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/MonkeyFactory.java +++ hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/MonkeyFactory.java @@ -51,21 +51,32 @@ public abstract class MonkeyFactory { public abstract ChaosMonkey build(); - + /* + * Factory Names + */ public static final String CALM = "calm"; public static final String SLOW_DETERMINISTIC = "slowDeterministic"; + public static final String SLOW_NO_MOVE = "slowNoMove"; + public static final String SLOW_NO_MERGE = "slowNoMerge"; + public static final String VERY_SLOW = "verySlow"; public static final String UNBALANCE = "unbalance"; + public static final String RESTART = "restart"; - public static Map FACTORIES = ImmutableMap.builder() - .put(CALM, new CalmMonkeyFactory()) - .put(SLOW_DETERMINISTIC, new SlowDeterministicMonkeyFactory()) - .put(UNBALANCE, new UnbalanceMonkeyFactory()) + public final static Map FACTORIES = ImmutableMap.builder() + .put(CALM.toLowerCase(), new CalmMonkeyFactory()) + .put(SLOW_DETERMINISTIC.toLowerCase(), new SlowDeterministicMonkeyFactory()) + .put(SLOW_NO_MERGE.toLowerCase(), new NoMergeSlowDeterministicMonkeyFactory()) + .put(SLOW_NO_MOVE.toLowerCase(), new NoMoveSlowDeterministicMonkeyFactory()) + .put(UNBALANCE.toLowerCase(), new UnbalanceMonkeyFactory()) + .put(VERY_SLOW.toLowerCase(), new VerySlowMonkeyFactory()) + .put(RESTART.toLowerCase(), new RestartMonkeyFactory()) .build(); public static MonkeyFactory getFactory(String factoryName) { - MonkeyFactory fact = FACTORIES.get(factoryName); + MonkeyFactory fact = FACTORIES.get(factoryName.toLowerCase()); if (fact == null) { - fact = FACTORIES.get(CALM); + fact = FACTORIES.get(CALM.toLowerCase()); + assert fact != null; } return fact; } diff --git hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/NoMergeSlowDeterministicMonkeyFactory.java hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/NoMergeSlowDeterministicMonkeyFactory.java new file mode 100644 index 0000000..f521735 --- /dev/null +++ hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/NoMergeSlowDeterministicMonkeyFactory.java @@ -0,0 +1,81 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.chaos.factories; + +import org.apache.hadoop.hbase.chaos.actions.Action; +import org.apache.hadoop.hbase.chaos.actions.BatchRestartRsAction; +import org.apache.hadoop.hbase.chaos.actions.CompactRandomRegionOfTableAction; +import org.apache.hadoop.hbase.chaos.actions.CompactTableAction; +import org.apache.hadoop.hbase.chaos.actions.FlushRandomRegionOfTableAction; +import org.apache.hadoop.hbase.chaos.actions.FlushTableAction; +import org.apache.hadoop.hbase.chaos.actions.MoveRandomRegionOfTableAction; +import org.apache.hadoop.hbase.chaos.actions.MoveRegionsOfTableAction; +import org.apache.hadoop.hbase.chaos.actions.RestartActiveMasterAction; +import org.apache.hadoop.hbase.chaos.actions.RestartRandomRsAction; +import org.apache.hadoop.hbase.chaos.actions.RestartRsHoldingMetaAction; +import org.apache.hadoop.hbase.chaos.actions.RollingBatchRestartRsAction; +import org.apache.hadoop.hbase.chaos.actions.SnapshotTableAction; +import org.apache.hadoop.hbase.chaos.actions.SplitRandomRegionOfTableAction; +import org.apache.hadoop.hbase.chaos.monkies.ChaosMonkey; +import org.apache.hadoop.hbase.chaos.monkies.PolicyBasedChaosMonkey; +import org.apache.hadoop.hbase.chaos.policies.CompositeSequentialPolicy; +import org.apache.hadoop.hbase.chaos.policies.DoActionsOncePolicy; +import org.apache.hadoop.hbase.chaos.policies.PeriodicRandomActionPolicy; + +public class NoMergeSlowDeterministicMonkeyFactory extends MonkeyFactory{ + @Override + public ChaosMonkey build() { + + // Actions such as compact/flush a table/region, + // move one region around. They are not so destructive, + // can be executed more frequently. + Action[] actions1 = new Action[] { + new CompactTableAction(tableName, 0.5f), + new CompactRandomRegionOfTableAction(tableName, 0.6f), + new FlushTableAction(tableName), + new FlushRandomRegionOfTableAction(tableName), + new MoveRandomRegionOfTableAction(tableName) + }; + + // Actions such as split/merge/snapshot. + // They should not cause data loss, or unreliability + // such as region stuck in transition. + Action[] actions2 = new Action[] { + new SplitRandomRegionOfTableAction(tableName), + new SnapshotTableAction(tableName) + }; + + // Destructive actions to mess things around. + Action[] actions3 = new Action[] { + new MoveRegionsOfTableAction(tableName), + new RestartRandomRsAction(60000), + new BatchRestartRsAction(5000, 0.5f), + new RestartActiveMasterAction(5000), + new RollingBatchRestartRsAction(5000, 1.0f), + new RestartRsHoldingMetaAction(35000) + }; + + return new PolicyBasedChaosMonkey(util, + new PeriodicRandomActionPolicy(60 * 1000, actions1), + new PeriodicRandomActionPolicy(90 * 1000, actions2), + new CompositeSequentialPolicy( + new DoActionsOncePolicy(150 * 1000, actions3), + new PeriodicRandomActionPolicy(150 * 1000, actions3))); + } +} diff --git hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/NoMoveSlowDeterministicMonkeyFactory.java hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/NoMoveSlowDeterministicMonkeyFactory.java new file mode 100644 index 0000000..0216262 --- /dev/null +++ hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/NoMoveSlowDeterministicMonkeyFactory.java @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.chaos.factories; + +import org.apache.hadoop.hbase.chaos.actions.Action; +import org.apache.hadoop.hbase.chaos.actions.BatchRestartRsAction; +import org.apache.hadoop.hbase.chaos.actions.CompactRandomRegionOfTableAction; +import org.apache.hadoop.hbase.chaos.actions.CompactTableAction; +import org.apache.hadoop.hbase.chaos.actions.FlushRandomRegionOfTableAction; +import org.apache.hadoop.hbase.chaos.actions.FlushTableAction; +import org.apache.hadoop.hbase.chaos.actions.MergeRandomAdjacentRegionsOfTableAction; +import org.apache.hadoop.hbase.chaos.actions.RestartActiveMasterAction; +import org.apache.hadoop.hbase.chaos.actions.RestartRandomRsAction; +import org.apache.hadoop.hbase.chaos.actions.RestartRsHoldingMetaAction; +import org.apache.hadoop.hbase.chaos.actions.RollingBatchRestartRsAction; +import org.apache.hadoop.hbase.chaos.actions.SnapshotTableAction; +import org.apache.hadoop.hbase.chaos.actions.SplitRandomRegionOfTableAction; +import org.apache.hadoop.hbase.chaos.monkies.ChaosMonkey; +import org.apache.hadoop.hbase.chaos.monkies.PolicyBasedChaosMonkey; +import org.apache.hadoop.hbase.chaos.policies.CompositeSequentialPolicy; +import org.apache.hadoop.hbase.chaos.policies.DoActionsOncePolicy; +import org.apache.hadoop.hbase.chaos.policies.PeriodicRandomActionPolicy; + +public class NoMoveSlowDeterministicMonkeyFactory extends MonkeyFactory { + @Override + public ChaosMonkey build() { + + // Actions such as compact/flush a table/region, + // move one region around. They are not so destructive, + // can be executed more frequently. + Action[] actions1 = new Action[] { + new CompactTableAction(tableName, 0.5f), + new CompactRandomRegionOfTableAction(tableName, 0.6f), + new FlushTableAction(tableName), + new FlushRandomRegionOfTableAction(tableName) + }; + + // Actions such as split/merge/snapshot. + // They should not cause data loss, or unreliability + // such as region stuck in transition. + Action[] actions2 = new Action[] { + new SplitRandomRegionOfTableAction(tableName), + new MergeRandomAdjacentRegionsOfTableAction(tableName), + new SnapshotTableAction(tableName) + }; + + // Destructive actions to mess things around. + Action[] actions3 = new Action[] { + new RestartRandomRsAction(60000), + new BatchRestartRsAction(5000, 0.5f), + new RestartActiveMasterAction(5000), + new RollingBatchRestartRsAction(5000, 1.0f), + new RestartRsHoldingMetaAction(35000) + }; + + return new PolicyBasedChaosMonkey(util, + new PeriodicRandomActionPolicy(60 * 1000, actions1), + new PeriodicRandomActionPolicy(90 * 1000, actions2), + new CompositeSequentialPolicy( + new DoActionsOncePolicy(150 * 1000, actions3), + new PeriodicRandomActionPolicy(150 * 1000, actions3))); + } +} diff --git hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/RestartMonkeyFactory.java hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/RestartMonkeyFactory.java new file mode 100644 index 0000000..ba87621 --- /dev/null +++ hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/RestartMonkeyFactory.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.chaos.factories; + +import org.apache.hadoop.hbase.chaos.actions.Action; +import org.apache.hadoop.hbase.chaos.actions.BatchRestartRsAction; +import org.apache.hadoop.hbase.chaos.actions.RestartActiveMasterAction; +import org.apache.hadoop.hbase.chaos.actions.RestartRandomRsAction; +import org.apache.hadoop.hbase.chaos.actions.RestartRsHoldingMetaAction; +import org.apache.hadoop.hbase.chaos.monkies.ChaosMonkey; +import org.apache.hadoop.hbase.chaos.monkies.PolicyBasedChaosMonkey; +import org.apache.hadoop.hbase.chaos.policies.PeriodicRandomActionPolicy; + +public class RestartMonkeyFactory extends MonkeyFactory{ + private static final long SLEEP_TIME = 60 * 1000; + + @Override + public ChaosMonkey build() { + Action[] actions = new Action[] { + new RestartRandomRsAction(SLEEP_TIME), + new BatchRestartRsAction(SLEEP_TIME, 0.5f), + new RestartActiveMasterAction(SLEEP_TIME), + new RestartRsHoldingMetaAction(SLEEP_TIME) + }; + + return new PolicyBasedChaosMonkey(util, new PeriodicRandomActionPolicy(SLEEP_TIME, actions)); + } +} diff --git hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/VerySlowMonkeyFactory.java hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/VerySlowMonkeyFactory.java new file mode 100644 index 0000000..8302001 --- /dev/null +++ hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/VerySlowMonkeyFactory.java @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.chaos.factories; + +import org.apache.hadoop.hbase.chaos.actions.Action; +import org.apache.hadoop.hbase.chaos.actions.BatchRestartRsAction; +import org.apache.hadoop.hbase.chaos.actions.CompactRandomRegionOfTableAction; +import org.apache.hadoop.hbase.chaos.actions.FlushRandomRegionOfTableAction; +import org.apache.hadoop.hbase.chaos.actions.MoveRegionsOfTableAction; +import org.apache.hadoop.hbase.chaos.actions.RestartActiveMasterAction; +import org.apache.hadoop.hbase.chaos.actions.RestartRandomRsAction; +import org.apache.hadoop.hbase.chaos.actions.RestartRsHoldingMetaAction; +import org.apache.hadoop.hbase.chaos.actions.SplitRandomRegionOfTableAction; +import org.apache.hadoop.hbase.chaos.monkies.ChaosMonkey; +import org.apache.hadoop.hbase.chaos.monkies.PolicyBasedChaosMonkey; +import org.apache.hadoop.hbase.chaos.policies.PeriodicRandomActionPolicy; + +/** + * Factory to create a very slow chaos monkey. + * + * This should be used with very long running tests (on the order of Hours.) + */ +public class VerySlowMonkeyFactory extends MonkeyFactory { + + public static final int SLEEP_TIME = 60*1000; + + @Override + public ChaosMonkey build() { + + Action[] actions1 = new Action[] { + new CompactRandomRegionOfTableAction(SLEEP_TIME, tableName, 0.5f), + new FlushRandomRegionOfTableAction(SLEEP_TIME, tableName), + new SplitRandomRegionOfTableAction(SLEEP_TIME, tableName) + }; + + Action[] actions2 = new Action[] { + new MoveRegionsOfTableAction(SLEEP_TIME, tableName), + new RestartRandomRsAction(SLEEP_TIME), + new BatchRestartRsAction(SLEEP_TIME, 0.5f), + new RestartActiveMasterAction(SLEEP_TIME) + }; + + return new PolicyBasedChaosMonkey(util, + new PeriodicRandomActionPolicy(SLEEP_TIME, actions1), + new PeriodicRandomActionPolicy(10 * SLEEP_TIME, actions2)); + } +} diff --git hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/monkies/ChaosMonkey.java hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/monkies/ChaosMonkey.java index da75c3b..d6e4281 100644 --- hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/monkies/ChaosMonkey.java +++ hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/monkies/ChaosMonkey.java @@ -18,7 +18,12 @@ package org.apache.hadoop.hbase.chaos.monkies; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.lang.math.RandomUtils; import org.apache.hadoop.hbase.Stoppable; +import org.apache.hadoop.hbase.util.Pair; /** * A utility to injects faults in a running cluster. @@ -51,4 +56,49 @@ public abstract class ChaosMonkey implements Stoppable { public abstract boolean isStopped(); public abstract void waitForStop() throws InterruptedException; + + /** Selects a random item from the given items */ + public static T selectRandomItem(T[] items) { + return items[RandomUtils.nextInt(items.length)]; + } + + /** Selects a random item from the given items with weights*/ + public static T selectWeightedRandomItem(List> items) { + int totalWeight = 0; + for (Pair pair : items) { + totalWeight += pair.getSecond(); + } + + int cutoff = RandomUtils.nextInt(totalWeight); + int cummulative = 0; + T item = null; + + //warn: O(n) + for (int i=0; i List selectRandomItems(T[] items, float ratio) { + int remaining = (int)Math.ceil(items.length * ratio); + + List selectedItems = new ArrayList(remaining); + + for (int i=0; i 0; i++) { + if (RandomUtils.nextFloat() < ((float)remaining/(items.length-i))) { + selectedItems.add(items[i]); + remaining--; + } + } + + return selectedItems; + } } diff --git hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/monkies/PolicyBasedChaosMonkey.java hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/monkies/PolicyBasedChaosMonkey.java index e267110..1bd8bbc 100644 --- hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/monkies/PolicyBasedChaosMonkey.java +++ hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/monkies/PolicyBasedChaosMonkey.java @@ -18,16 +18,12 @@ package org.apache.hadoop.hbase.chaos.monkies; -import java.util.ArrayList; import java.util.Collection; -import java.util.List; -import org.apache.commons.lang.math.RandomUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.IntegrationTestingUtility; import org.apache.hadoop.hbase.chaos.policies.Policy; -import org.apache.hadoop.hbase.util.Pair; /** * Chaos monkey that given multiple policies will run actions against the cluster. @@ -59,51 +55,6 @@ public class PolicyBasedChaosMonkey extends ChaosMonkey { } - /** Selects a random item from the given items */ - public static T selectRandomItem(T[] items) { - return items[RandomUtils.nextInt(items.length)]; - } - - /** Selects a random item from the given items with weights*/ - public static T selectWeightedRandomItem(List> items) { - int totalWeight = 0; - for (Pair pair : items) { - totalWeight += pair.getSecond(); - } - - int cutoff = RandomUtils.nextInt(totalWeight); - int cummulative = 0; - T item = null; - - //warn: O(n) - for (int i=0; i List selectRandomItems(T[] items, float ratio) { - int remaining = (int)Math.ceil(items.length * ratio); - - List selectedItems = new ArrayList(remaining); - - for (int i=0; i 0; i++) { - if (RandomUtils.nextFloat() < ((float)remaining/(items.length-i))) { - selectedItems.add(items[i]); - remaining--; - } - } - - return selectedItems; - } - private Policy[] policies; private Thread[] monkeyThreads; diff --git hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/policies/PeriodicRandomActionPolicy.java hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/policies/PeriodicRandomActionPolicy.java index 8912467..d624e9f 100644 --- hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/policies/PeriodicRandomActionPolicy.java +++ hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/policies/PeriodicRandomActionPolicy.java @@ -23,7 +23,7 @@ import java.util.Arrays; import java.util.List; import org.apache.hadoop.hbase.chaos.actions.Action; -import org.apache.hadoop.hbase.chaos.monkies.PolicyBasedChaosMonkey; +import org.apache.hadoop.hbase.chaos.monkies.ChaosMonkey; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.util.StringUtils; @@ -54,7 +54,7 @@ public class PeriodicRandomActionPolicy extends PeriodicPolicy { @Override protected void runOneIteration() { - Action action = PolicyBasedChaosMonkey.selectWeightedRandomItem(actions); + Action action = ChaosMonkey.selectWeightedRandomItem(actions); try { action.perform(); } catch (Exception ex) { diff --git hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java index b1956df..ad36c19 100644 --- hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java +++ hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java @@ -54,6 +54,7 @@ import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.ScannerCallable; +import org.apache.hadoop.hbase.filter.KeyOnlyFilter; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; import org.apache.hadoop.hbase.mapreduce.TableMapper; @@ -82,6 +83,8 @@ import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat; import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -332,7 +335,7 @@ public class IntegrationTestBigLinkedList extends IntegrationTestBase { int width; protected void setup(Context context) throws IOException, InterruptedException { - id = Bytes.toBytes(UUID.randomUUID().toString()); + id = Bytes.toBytes("Job: "+context.getJobID() + " Task: " + context.getTaskAttemptID()); Configuration conf = context.getConfiguration(); table = new HTable(conf, getTableName(conf)); table.setAutoFlush(false); diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/HLogInputFormat.java hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/HLogInputFormat.java index 7fa852e..f04896d 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/HLogInputFormat.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/HLogInputFormat.java @@ -57,7 +57,7 @@ public class HLogInputFormat extends InputFormat { * {@link InputSplit} for {@link HLog} files. Each split represent * exactly one log file. */ - static class HLogSplit extends InputSplit implements Writable { + public static class HLogSplit extends InputSplit implements Writable { private String logFileName; private long fileSize; private long startTime; diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogPrettyPrinter.java hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogPrettyPrinter.java index c2138a9..180ba9c 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogPrettyPrinter.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogPrettyPrinter.java @@ -282,8 +282,7 @@ public class HLogPrettyPrinter { out.println("Sequence " + txn.get("sequence") + " " + "from region " + txn.get("region") + " " + "in table " + txn.get("table") + " at write timestamp: " + new Date(writeTime)); - for (int i = 0; i < actions.size(); i++) { - Map op = actions.get(i); + for (Map op : actions) { out.println(" Action:"); out.println(" row: " + op.get("row")); out.println(" column: " + op.get("family") + ":" diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/util/RowLogSearchJob.java hbase-server/src/main/java/org/apache/hadoop/hbase/util/RowLogSearchJob.java new file mode 100644 index 0000000..82abe0c --- /dev/null +++ hbase-server/src/main/java/org/apache/hadoop/hbase/util/RowLogSearchJob.java @@ -0,0 +1,120 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.io.IOException; + +import org.apache.commons.cli.CommandLine; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.mapreduce.HLogInputFormat; +import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; +import org.apache.hadoop.hbase.regionserver.wal.HLogKey; +import org.apache.hadoop.hbase.regionserver.wal.WALEdit; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.Mapper; +import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; + +/** + * Class to search for rows from a set of WAL logs. + */ +public class RowLogSearchJob extends AbstractHBaseTool { + + public String inputPath = null; + public String outputPath = null; + + public static String[] MISSING_ROWS = { + "\\x8E0\\x05\\x0E\\x83yn6\\xC5\\xF38&\\xDB\\xA10\\xE8", + "\\x8E2R\\xA5\\xCE\\xC5r\\xFD\\xCDbs\\xA3\\xC3\\xF9B]", + "\\x8E4[\\xBA\\x02\\xB2\\xB6(KX\\xB3\\xBFR\\xCD\\xCB\\x03", + "\\x8E6\\x98\\xFCj\\xDE%LAbl\\x7F\\x03\\xEE\\xDEO", + "\\x8E<\\xF3\\xE1f\\x1Al\\x89d\\xD3\\xC1w5\\x9B\\x8FN" + }; + + public static class RowLogSearchJobMapper extends Mapper { + protected void map(HLogKey key, WALEdit value, Context context) throws IOException, InterruptedException { + String fileName = + ((HLogInputFormat.HLogSplit) context.getInputSplit()).getLogFileName(); + Text textKey = new Text(fileName); + + + for (KeyValue kv:value.getKeyValues()) { + String rkString = Bytes.toStringBinary(kv.getRow()); + for (String missingRK:MISSING_ROWS) { + if (rkString.equals(missingRK)) { + context.write(textKey, NullWritable.get()); + } + } + } + } + } + + @Override + protected void addOptions() { + + } + + @Override + protected void processOptions(CommandLine cmd) { + inputPath = cmd.getArgs()[0]; + outputPath = cmd.getArgs()[1]; + + } + + @Override + protected int doWork() throws Exception { + Job job = Job.getInstance(getConf(), "Row Log Search"); + job.setJarByClass(getClass()); + + TextOutputFormat.setOutputPath(job, new Path(outputPath)); + job.getConfiguration().set("mapred.input.dir", inputPath); + + TableMapReduceUtil.addDependencyJars(job); + TableMapReduceUtil.addDependencyJars(job.getConfiguration(), + AbstractHBaseTool.class, KeyValue.class, HLogKey.class, HLogInputFormat.class); + + job.getConfiguration().setBoolean("mapred.map.tasks.speculative.execution", false); + job.getConfiguration().setBoolean("mapred.reduce.tasks.speculative.execution", false); + + job.setInputFormatClass(HLogInputFormat.class); + job.setMapperClass(RowLogSearchJobMapper.class); + + job.setMapOutputKeyClass(Text.class); + job.setMapOutputValueClass(NullWritable.class); + + job.setReduceSpeculativeExecution(false); + job.setMapSpeculativeExecution(false); + + job.setOutputFormatClass(TextOutputFormat.class); + + + + return job.waitForCompletion(true)?0:1; + } + + public static void main(String[] args) throws Exception { + Tool t = new RowLogSearchJob(); + + ToolRunner.run(HBaseConfiguration.create(), t, args); + } +} -- 1.7.10.2 (Apple Git-33)