diff --git a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java index 678e02a..030e6c6 100644 --- a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java +++ b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java @@ -2015,6 +2015,28 @@ public class HBaseAdmin implements Abortable, Closeable { } /** + * Create snapshot for the given table of given flush type. + *

+ * Snapshots are considered unique based on the name of the snapshot. Attempts to take a + * snapshot with the same name (even a different type or with different parameters) will fail with + * a {@link SnapshotCreationException} indicating the duplicate naming. + *

+ * Snapshot names follow the same naming constraints as tables in HBase. See + * {@link HTableDescriptor#isLegalTableName(byte[])}. + * @param snapshotName name of the snapshot to be created + * @param tableName name of the table for which snapshot is created + * @param flushType if the snapshot should be taken without flush memstore first + * @throws IOException if a remote or network exception occurs + * @throws SnapshotCreationException if snapshot creation failed + * @throws IllegalArgumentException if the snapshot request is formatted incorrectly + */ + public void snapshot(final byte[] snapshotName, final byte[] tableName, + final SnapshotDescription.Type flushType) throws + IOException, SnapshotCreationException, IllegalArgumentException { + snapshot(Bytes.toString(snapshotName), Bytes.toString(tableName), flushType); + } + + /** * Take a snapshot for the given table. If the table is enabled, a FLUSH-type snapshot will be * taken. If the table is disabled, an offline snapshot is taken. *

diff --git a/src/main/java/org/apache/hadoop/hbase/protobuf/generated/HBaseProtos.java b/src/main/java/org/apache/hadoop/hbase/protobuf/generated/HBaseProtos.java index e0ccb1c..948f73a 100644 --- a/src/main/java/org/apache/hadoop/hbase/protobuf/generated/HBaseProtos.java +++ b/src/main/java/org/apache/hadoop/hbase/protobuf/generated/HBaseProtos.java @@ -63,10 +63,12 @@ public final class HBaseProtos { implements com.google.protobuf.ProtocolMessageEnum { DISABLED(0, 0), FLUSH(1, 1), + SKIPFLUSH(2, 2), ; public static final int DISABLED_VALUE = 0; public static final int FLUSH_VALUE = 1; + public static final int SKIPFLUSH_VALUE = 2; public final int getNumber() { return value; } @@ -75,6 +77,7 @@ public final class HBaseProtos { switch (value) { case 0: return DISABLED; case 1: return FLUSH; + case 2: return SKIPFLUSH; default: return null; } } @@ -105,7 +108,7 @@ public final class HBaseProtos { } private static final Type[] VALUES = { - DISABLED, FLUSH, + DISABLED, FLUSH, SKIPFLUSH }; public static Type valueOf( diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/FlushSnapshotSubprocedure.java b/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/FlushSnapshotSubprocedure.java index b5d194c..9917ba9 100644 --- a/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/FlushSnapshotSubprocedure.java +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/FlushSnapshotSubprocedure.java @@ -49,12 +49,18 @@ public class FlushSnapshotSubprocedure extends Subprocedure { private final SnapshotDescription snapshot; private final SnapshotSubprocedurePool taskManager; + private boolean snapshotSkipFlush = false; + public FlushSnapshotSubprocedure(ProcedureMember member, ForeignExceptionDispatcher errorListener, long wakeFrequency, long timeout, List regions, SnapshotDescription snapshot, SnapshotSubprocedurePool taskManager) { super(member, snapshot.getName(), errorListener, wakeFrequency, timeout); this.snapshot = snapshot; + if (this.snapshot.getType() == SnapshotDescription.Type.SKIPFLUSH) { + snapshotSkipFlush = true; + } + this.regions = regions; this.taskManager = taskManager; } @@ -78,10 +84,25 @@ public class FlushSnapshotSubprocedure extends Subprocedure { LOG.debug("Starting region operation on " + region); region.startRegionOperation(); try { - LOG.debug("Flush Snapshotting region " + region.toString() + " started..."); - region.flushcache(); + if (snapshotSkipFlush) { + /* + * This is to take an online-snapshot without force a coordinated flush to prevent pause + * The snapshot type is defined inside the snapshot description. FlushSnapshotSubprocedure + * should be renamed to distributedSnapshotSubprocedure, and the flush() behavior can be + * turned on/off based on the flush type. + * To minimized the code change, class name is not changed. + */ + LOG.debug("take snapshot without flush memstore first"); + } else { + LOG.debug("Flush Snapshotting region " + region.toString() + " started..."); + region.flushcache(); + } region.addRegionToSnapshot(snapshot, monitor); - LOG.debug("... Flush Snapshotting region " + region.toString() + " completed."); + if (snapshotSkipFlush) { + LOG.debug("... SkipFlush Snapshotting region " + region.toString() + " completed."); + } else { + LOG.debug("... Flush Snapshotting region " + region.toString() + " completed."); + } } finally { LOG.debug("Closing region operation on " + region); region.closeRegionOperation(); diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/RegionServerSnapshotManager.java b/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/RegionServerSnapshotManager.java index 60e23df..4eab185 100644 --- a/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/RegionServerSnapshotManager.java +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/RegionServerSnapshotManager.java @@ -204,6 +204,18 @@ public class RegionServerSnapshotManager { new SnapshotSubprocedurePool(rss.getServerName().toString(), conf); return new FlushSnapshotSubprocedure(member, exnDispatcher, wakeMillis, timeoutMillis, involvedRegions, snapshot, taskManager); + case SKIPFLUSH: + /* + * This is to take an online-snapshot without force a coordinated flush to prevent pause + * The snapshot type is defined inside the snapshot description. FlushSnapshotSubprocedure + * should be renamed to distributedSnapshotSubprocedure, and the flush() behavior can be + * turned on/off based on the flush type. + * To minimized the code change, class name is not changed. + */ + SnapshotSubprocedurePool taskManager2 = + new SnapshotSubprocedurePool(rss.getServerName().toString(), conf); + return new FlushSnapshotSubprocedure(member, exnDispatcher, wakeMillis, + timeoutMillis, involvedRegions, snapshot, taskManager2); default: throw new UnsupportedOperationException("Unrecognized snapshot type:" + snapshot.getType()); } diff --git a/src/main/protobuf/hbase.proto b/src/main/protobuf/hbase.proto index c3c97ad..0cdeaeb 100644 --- a/src/main/protobuf/hbase.proto +++ b/src/main/protobuf/hbase.proto @@ -33,6 +33,7 @@ message SnapshotDescription { enum Type { DISABLED = 0; FLUSH = 1; + SKIPFLUSH = 2; } optional Type type = 4 [default = FLUSH]; optional int32 version = 5; diff --git a/src/main/ruby/hbase.rb b/src/main/ruby/hbase.rb index 4d97cd0..fbcdc00 100644 --- a/src/main/ruby/hbase.rb +++ b/src/main/ruby/hbase.rb @@ -57,6 +57,7 @@ module HBaseConstants SPLITS_FILE = 'SPLITS_FILE' SPLITALGO = 'SPLITALGO' NUMREGIONS = 'NUMREGIONS' + SKIP_FLUSH = 'SKIP_FLUSH' # Load constants from hbase java API def self.promote_constants(constants) diff --git a/src/main/ruby/hbase/admin.rb b/src/main/ruby/hbase/admin.rb index 4ad3d88..3ccb7e3 100644 --- a/src/main/ruby/hbase/admin.rb +++ b/src/main/ruby/hbase/admin.rb @@ -21,6 +21,7 @@ include Java java_import org.apache.hadoop.hbase.util.Pair java_import org.apache.hadoop.hbase.util.RegionSplitter +java_import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos::SnapshotDescription # Wrapper for org.apache.hadoop.hbase.client.HBaseAdmin @@ -636,8 +637,18 @@ module Hbase #---------------------------------------------------------------------------------------------- # Take a snapshot of specified table - def snapshot(table, snapshot_name) - @admin.snapshot(snapshot_name.to_java_bytes, table.to_java_bytes) + def snapshot(table, snapshot_name, *args) + if args.empty? + @admin.snapshot(snapshot_name.to_java_bytes, table.to_java_bytes) + else + args.each do |arg| + if arg[SKIP_FLUSH] == true + @admin.snapshot(snapshot_name.to_java_bytes, table.to_java_bytes, SnapshotDescription::Type::SKIPFLUSH) + else + @admin.snapshot(snapshot_name.to_java_bytes, table.to_java_bytes) + end + end + end end #---------------------------------------------------------------------------------------------- diff --git a/src/main/ruby/shell/commands/snapshot.rb b/src/main/ruby/shell/commands/snapshot.rb index 1c4ecfe..899789e 100644 --- a/src/main/ruby/shell/commands/snapshot.rb +++ b/src/main/ruby/shell/commands/snapshot.rb @@ -24,12 +24,13 @@ module Shell Take a snapshot of specified table. Examples: hbase> snapshot 'sourceTable', 'snapshotName' + hbase> snapshot 'sourceTable', 'snapshotName', {SKIP_FLUSH => true} EOF end - def command(table, snapshot_name) + def command(table, snapshot_name, *args) format_simple_command do - admin.snapshot(table, snapshot_name) + admin.snapshot(table, snapshot_name, *args) end end end diff --git a/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java b/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java index 5dfd7eb..c7a265f 100644 --- a/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java +++ b/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java @@ -126,6 +126,57 @@ public class TestFlushSnapshotFromClient { } /** + * Test snapshotting a table that is online without flushing + * @throws Exception + */ + @Test + public void testSkipFlushTableSnapshot() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + // make sure we don't fail on listing snapshots + SnapshotTestingUtils.assertNoSnapshots(admin); + + // put some stuff in the table + HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME); + UTIL.loadTable(table, TEST_FAM); + + // get the name of all the regionservers hosting the snapshotted table + Set snapshotServers = new HashSet(); + List servers = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads(); + for (RegionServerThread server : servers) { + if (server.getRegionServer().getOnlineRegions(TABLE_NAME).size() > 0) { + snapshotServers.add(server.getRegionServer().getServerName().toString()); + } + } + + LOG.debug("FS state before snapshot:"); + FSUtils.logFileSystemState(UTIL.getTestFileSystem(), + FSUtils.getRootDir(UTIL.getConfiguration()), LOG); + + // take a snapshot of the enabled table + String snapshotString = "skipFlushTableSnapshot"; + byte[] snapshot = Bytes.toBytes(snapshotString); + admin.snapshot(snapshotString, STRING_TABLE_NAME, SnapshotDescription.Type.SKIPFLUSH); + LOG.debug("Snapshot completed."); + + // make sure we have the snapshot + List snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, + snapshot, TABLE_NAME); + + // make sure its a valid snapshot + FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem(); + Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir(); + LOG.debug("FS state after snapshot:"); + FSUtils.logFileSystemState(UTIL.getTestFileSystem(), + FSUtils.getRootDir(UTIL.getConfiguration()), LOG); + + SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir, + admin, fs, false, new Path(rootDir, HConstants.HREGION_LOGDIR_NAME), snapshotServers); + + admin.deleteSnapshot(snapshot); + snapshots = admin.listSnapshots(); + SnapshotTestingUtils.assertNoSnapshots(admin); + } + /** * Test simple flush snapshotting a table that is online * @throws Exception */