From 9bfff66786af62cf919903f1e0901c4055fe41e2 Mon Sep 17 00:00:00 2001 From: ted malaska Date: Sat, 23 May 2015 22:22:56 -0400 Subject: [PATCH] HBASE-12481 moved tools out of HRegion and FSHLog --- .../CatalogTableCompactorDumperTool.java | 145 +++++++++++++++++++++ .../apache/hadoop/hbase/regionserver/HRegion.java | 102 +-------------- .../hadoop/hbase/regionserver/wal/FSHLog.java | 65 --------- .../hbase/regionserver/wal/LogFileDumperTool.java | 105 +++++++++++++++ 4 files changed, 251 insertions(+), 166 deletions(-) create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CatalogTableCompactorDumperTool.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/LogFileDumperTool.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CatalogTableCompactorDumperTool.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CatalogTableCompactorDumperTool.java new file mode 100644 index 0000000..eebc2ea --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CatalogTableCompactorDumperTool.java @@ -0,0 +1,145 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.io.hfile.BlockCache; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; +import org.apache.hadoop.hbase.util.FSTableDescriptors; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hbase.wal.WAL; +import org.apache.hadoop.hbase.wal.WALFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class CatalogTableCompactorDumperTool { + + private static final Log LOG = LogFactory.getLog(CatalogTableCompactorDumperTool.class); + + /** + * Facility for dumping and compacting catalog tables. + * Only does catalog tables since these are only tables we for sure know + * schema on. For usage run: + *
+   *   ./bin/hbase org.apache.hadoop.hbase.regionserver.CatalogTableCompactorDumperTool
+   * 
+ * @throws java.io.IOException + */ + public static void main(String[] args) throws IOException { + if (args.length < 1) { + printUsageAndExit(null); + } + boolean majorCompact = false; + if (args.length > 1) { + if (!args[1].toLowerCase().startsWith("major")) { + printUsageAndExit("ERROR: Unrecognized option <" + args[1] + ">"); + } + majorCompact = true; + } + final Path tableDir = new Path(args[0]); + final Configuration c = HBaseConfiguration.create(); + final FileSystem fs = FileSystem.get(c); + final Path logdir = new Path(c.get("hbase.tmp.dir")); + final String logname = "wal" + FSUtils.getTableName(tableDir) + System.currentTimeMillis(); + + final Configuration walConf = new Configuration(c); + FSUtils.setRootDir(walConf, logdir); + final WALFactory wals = new WALFactory(walConf, null, logname); + try { + processTable(fs, tableDir, wals, c, majorCompact); + } finally { + wals.close(); + // TODO: is this still right? + BlockCache bc = new CacheConfig(c).getBlockCache(); + if (bc != null) bc.shutdown(); + } + } + + /* + * Process table. + * Do major compaction or list content. + * @throws IOException + */ + private static void processTable(final FileSystem fs, final Path p, + final WALFactory walFactory, final Configuration c, + final boolean majorCompact) + throws IOException { + HRegion region; + FSTableDescriptors fst = new FSTableDescriptors(c); + // Currently expects tables have one region only. + if (FSUtils.getTableName(p).equals(TableName.META_TABLE_NAME)) { + final WAL wal = walFactory.getMetaWAL( + HRegionInfo.FIRST_META_REGIONINFO.getEncodedNameAsBytes()); + region = HRegion.newHRegion(p, wal, fs, c, + HRegionInfo.FIRST_META_REGIONINFO, + fst.get(TableName.META_TABLE_NAME), null); + } else { + throw new IOException("Not a known catalog table: " + p.toString()); + } + try { + region.initialize(null); + if (majorCompact) { + region.compact(true); + } else { + // Default behavior + Scan scan = new Scan(); + // scan.addFamily(HConstants.CATALOG_FAMILY); + RegionScanner scanner = region.getScanner(scan); + try { + List kvs = new ArrayList(); + boolean done; + do { + kvs.clear(); + done = scanner.next(kvs); + if (kvs.size() > 0) LOG.info(kvs); + } while (done); + } finally { + scanner.close(); + } + } + } finally { + region.close(); + } + } + + /* + * This method calls System.exit. + * @param message Message to print out. May be null. + */ + private static void printUsageAndExit(final String message) { + if (message != null && message.length() > 0) System.out.println(message); + System.out.println("Usage: HRegion CATALOG_TABLE_DIR [major_compact]"); + System.out.println("Options:"); + System.out.println(" major_compact Pass this option to major compact " + + "passed region."); + System.out.println("Default outputs scan of passed region."); + System.exit(1); + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java index 3994027..e208073 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -765,7 +765,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi * @return What the next sequence (edit) id should be. * @throws IOException e */ - private long initialize(final CancelableProgressable reporter) throws IOException { + long initialize(final CancelableProgressable reporter) throws IOException { MonitoredTask status = TaskMonitor.get().createStatus("Initializing region " + this); long nextSeqId = -1; try { @@ -7216,20 +7216,6 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi return heapSize; } - /* - * This method calls System.exit. - * @param message Message to print out. May be null. - */ - private static void printUsageAndExit(final String message) { - if (message != null && message.length() > 0) System.out.println(message); - System.out.println("Usage: HRegion CATALOG_TABLE_DIR [major_compact]"); - System.out.println("Options:"); - System.out.println(" major_compact Pass this option to major compact " + - "passed region."); - System.out.println("Default outputs scan of passed region."); - System.exit(1); - } - @Override public boolean registerService(Service instance) { /* @@ -7302,53 +7288,6 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi return responseBuilder.build(); } - /* - * Process table. - * Do major compaction or list content. - * @throws IOException - */ - private static void processTable(final FileSystem fs, final Path p, - final WALFactory walFactory, final Configuration c, - final boolean majorCompact) - throws IOException { - HRegion region; - FSTableDescriptors fst = new FSTableDescriptors(c); - // Currently expects tables have one region only. - if (FSUtils.getTableName(p).equals(TableName.META_TABLE_NAME)) { - final WAL wal = walFactory.getMetaWAL( - HRegionInfo.FIRST_META_REGIONINFO.getEncodedNameAsBytes()); - region = HRegion.newHRegion(p, wal, fs, c, - HRegionInfo.FIRST_META_REGIONINFO, - fst.get(TableName.META_TABLE_NAME), null); - } else { - throw new IOException("Not a known catalog table: " + p.toString()); - } - try { - region.initialize(null); - if (majorCompact) { - region.compact(true); - } else { - // Default behavior - Scan scan = new Scan(); - // scan.addFamily(HConstants.CATALOG_FAMILY); - RegionScanner scanner = region.getScanner(scan); - try { - List kvs = new ArrayList(); - boolean done; - do { - kvs.clear(); - done = scanner.next(kvs); - if (kvs.size() > 0) LOG.info(kvs); - } while (done); - } finally { - scanner.close(); - } - } - } finally { - region.close(); - } - } - boolean shouldForceSplit() { return this.splitRequest; } @@ -7662,45 +7601,6 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi } }; - /** - * Facility for dumping and compacting catalog tables. - * Only does catalog tables since these are only tables we for sure know - * schema on. For usage run: - *
-   *   ./bin/hbase org.apache.hadoop.hbase.regionserver.HRegion
-   * 
- * @throws IOException - */ - public static void main(String[] args) throws IOException { - if (args.length < 1) { - printUsageAndExit(null); - } - boolean majorCompact = false; - if (args.length > 1) { - if (!args[1].toLowerCase().startsWith("major")) { - printUsageAndExit("ERROR: Unrecognized option <" + args[1] + ">"); - } - majorCompact = true; - } - final Path tableDir = new Path(args[0]); - final Configuration c = HBaseConfiguration.create(); - final FileSystem fs = FileSystem.get(c); - final Path logdir = new Path(c.get("hbase.tmp.dir")); - final String logname = "wal" + FSUtils.getTableName(tableDir) + System.currentTimeMillis(); - - final Configuration walConf = new Configuration(c); - FSUtils.setRootDir(walConf, logdir); - final WALFactory wals = new WALFactory(walConf, null, logname); - try { - processTable(fs, tableDir, wals, c, majorCompact); - } finally { - wals.close(); - // TODO: is this still right? - BlockCache bc = new CacheConfig(c).getBlockCache(); - if (bc != null) bc.shutdown(); - } - } - @Override public long getOpenSeqNum() { return this.openSeqNum; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java index 549f0ce..36407c9 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java @@ -1743,22 +1743,6 @@ public class FSHLog implements WAL { ClassSize.OBJECT + (5 * ClassSize.REFERENCE) + ClassSize.ATOMIC_INTEGER + Bytes.SIZEOF_INT + (3 * Bytes.SIZEOF_LONG)); - private static void split(final Configuration conf, final Path p) - throws IOException { - FileSystem fs = FileSystem.get(conf); - if (!fs.exists(p)) { - throw new FileNotFoundException(p.toString()); - } - if (!fs.getFileStatus(p).isDirectory()) { - throw new IOException(p + " is not a directory"); - } - - final Path baseDir = FSUtils.getRootDir(conf); - final Path archiveDir = new Path(baseDir, HConstants.HREGION_OLDLOGDIR_NAME); - WALSplitter.split(baseDir, p, archiveDir, fs, conf, WALFactory.getInstance(conf)); - } - - @Override public long getEarliestMemstoreSeqNum(byte[] encodedRegionName) { ConcurrentMap oldestUnflushedStoreSequenceIdsOfRegion = @@ -2107,55 +2091,6 @@ public class FSHLog implements WAL { return (t instanceof IOException)? (IOException)t: new IOException(t); } - private static void usage() { - System.err.println("Usage: FSHLog "); - System.err.println("Arguments:"); - System.err.println(" --dump Dump textual representation of passed one or more files"); - System.err.println(" For example: " + - "FSHLog --dump hdfs://example.com:9000/hbase/.logs/MACHINE/LOGFILE"); - System.err.println(" --split Split the passed directory of WAL logs"); - System.err.println(" For example: " + - "FSHLog --split hdfs://example.com:9000/hbase/.logs/DIR"); - } - - /** - * Pass one or more log file names and it will either dump out a text version - * on stdout or split the specified log files. - * - * @param args - * @throws IOException - */ - public static void main(String[] args) throws IOException { - if (args.length < 2) { - usage(); - System.exit(-1); - } - // either dump using the WALPrettyPrinter or split, depending on args - if (args[0].compareTo("--dump") == 0) { - WALPrettyPrinter.run(Arrays.copyOfRange(args, 1, args.length)); - } else if (args[0].compareTo("--perf") == 0) { - LOG.fatal("Please use the WALPerformanceEvaluation tool instead. i.e.:"); - LOG.fatal("\thbase org.apache.hadoop.hbase.wal.WALPerformanceEvaluation --iterations " + - args[1]); - System.exit(-1); - } else if (args[0].compareTo("--split") == 0) { - Configuration conf = HBaseConfiguration.create(); - for (int i = 1; i < args.length; i++) { - try { - Path logPath = new Path(args[i]); - FSUtils.setFsDefault(conf, logPath); - split(conf, logPath); - } catch (IOException t) { - t.printStackTrace(System.err); - System.exit(-1); - } - } - } else { - usage(); - System.exit(-1); - } - } - /** * Find the 'getPipeline' on the passed os stream. * @return Method or null. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/LogFileDumperTool.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/LogFileDumperTool.java new file mode 100644 index 0000000..d0e204f --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/LogFileDumperTool.java @@ -0,0 +1,105 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.wal; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hbase.wal.WALFactory; +import org.apache.hadoop.hbase.wal.WALPrettyPrinter; +import org.apache.hadoop.hbase.wal.WALSplitter; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.Arrays; + +public class LogFileDumperTool { + + private static final Log LOG = LogFactory.getLog(LogFileDumperTool.class); + + /** + * Pass one or more log file names and it will either dump out a text version + * on stdout or split the specified log files. + * + * @param args + * @throws java.io.IOException + */ + public static void main(String[] args) throws IOException { + if (args.length < 2) { + usage(); + System.exit(-1); + } + // either dump using the WALPrettyPrinter or split, depending on args + if (args[0].compareTo("--dump") == 0) { + WALPrettyPrinter.run(Arrays.copyOfRange(args, 1, args.length)); + } else if (args[0].compareTo("--perf") == 0) { + LOG.fatal("Please use the WALPerformanceEvaluation tool instead. i.e.:"); + LOG.fatal("\thbase org.apache.hadoop.hbase.wal.WALPerformanceEvaluation --iterations " + + args[1]); + System.exit(-1); + } else if (args[0].compareTo("--split") == 0) { + Configuration conf = HBaseConfiguration.create(); + for (int i = 1; i < args.length; i++) { + try { + Path logPath = new Path(args[i]); + FSUtils.setFsDefault(conf, logPath); + split(conf, logPath); + } catch (IOException t) { + t.printStackTrace(System.err); + System.exit(-1); + } + } + } else { + usage(); + System.exit(-1); + } + } + + private static void split(final Configuration conf, final Path p) + throws IOException { + FileSystem fs = FileSystem.get(conf); + if (!fs.exists(p)) { + throw new FileNotFoundException(p.toString()); + } + if (!fs.getFileStatus(p).isDirectory()) { + throw new IOException(p + " is not a directory"); + } + + final Path baseDir = FSUtils.getRootDir(conf); + final Path archiveDir = new Path(baseDir, HConstants.HREGION_OLDLOGDIR_NAME); + WALSplitter.split(baseDir, p, archiveDir, fs, conf, WALFactory.getInstance(conf)); + } + + private static void usage() { + System.err.println("Usage: FSHLog "); + System.err.println("Arguments:"); + System.err.println(" --dump Dump textual representation of passed one or more files"); + System.err.println(" For example: " + + "FSHLog --dump hdfs://example.com:9000/hbase/.logs/MACHINE/LOGFILE"); + System.err.println(" --split Split the passed directory of WAL logs"); + System.err.println(" For example: " + + "FSHLog --split hdfs://example.com:9000/hbase/.logs/DIR"); + } + + +} -- 2.3.2 (Apple Git-55)