From 2780cb8b1d5167f9faacabdab95714a1829ab685 Mon Sep 17 00:00:00 2001 From: stack Date: Wed, 13 Nov 2019 15:16:45 -0800 Subject: [PATCH] HBASE-23291 ProcedureStoreReader Tool --- .../hbase/util/WALProcedureStoreReader.java | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 hbase-examples/src/main/java/org/apache/hadoop/hbase/util/WALProcedureStoreReader.java diff --git a/hbase-examples/src/main/java/org/apache/hadoop/hbase/util/WALProcedureStoreReader.java b/hbase-examples/src/main/java/org/apache/hadoop/hbase/util/WALProcedureStoreReader.java new file mode 100644 index 0000000000..ddddb99551 --- /dev/null +++ b/hbase-examples/src/main/java/org/apache/hadoop/hbase/util/WALProcedureStoreReader.java @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.master.MasterServices; +import org.apache.hadoop.hbase.master.procedure.MasterProcedureConstants; +import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; +import org.apache.hadoop.hbase.master.procedure.MasterProcedureScheduler; +import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; +import org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore; +import org.apache.yetus.audience.InterfaceAudience; +import org.mockito.Mockito; + +import java.io.IOException; + +/** + * Needs to be in a module named hbase-tools, not in hbase-examples. + */ +@InterfaceAudience.Private +public class WALProcedureStoreReader { + /** + * Parses a directory of WALs building up ProcedureState. + * For testing parse and profiling. + * @param args Include pointer to directory of WAL files for a store instance to parse & load. + */ + public static void main(String [] args) throws IOException { + Configuration conf = HBaseConfiguration.create(); + if (args == null || args.length != 1) { + System.out.println("ERROR: Empty arguments list; pass path to MASTERPROCWALS_DIR."); + System.out.println("Usage: WALProcedureStore MASTERPROCWALS_DIR"); + System.exit(-1); + } + WALProcedureStore store = new WALProcedureStore(conf, new Path(args[0]), null, + new WALProcedureStore.LeaseRecovery() { + @Override + public void recoverFileLease(FileSystem fs, Path path) throws IOException { + // no-op + } + }); + try { + MasterServices masterServices = Mockito.mock(MasterServices.class); + Mockito.when(masterServices.getConfiguration()).thenReturn(conf); + MasterProcedureEnv env = new MasterProcedureEnv(masterServices); + MasterProcedureScheduler procedureScheduler = env.getProcedureScheduler(); + ProcedureExecutor procedureExecutor = + new ProcedureExecutor<>(conf, env, store, procedureScheduler); + Mockito.when(masterServices.getMasterProcedureExecutor()).thenReturn(procedureExecutor); + int numThreads = getNumberOfThreads(conf); + store.start(numThreads); + procedureExecutor.init(numThreads, false); + } finally { + store.stop(true); + } + } + + private static int getNumberOfThreads(Configuration conf) { + int cpus = Runtime.getRuntime().availableProcessors(); + return conf.getInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, Math.max( + (cpus > 0 ? cpus / 4 : 0), MasterProcedureConstants.DEFAULT_MIN_MASTER_PROCEDURE_THREADS)); + } +} -- 2.19.1