diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java index 759822a..2d56823 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java @@ -2613,6 +2613,27 @@ public class HBaseAdmin implements Admin { } /** + * Create snapshot for the given table of given flush type. + *

+ * Snapshots are considered unique based on the name of the snapshot. Attempts to take a + * snapshot with the same name (even a different type or with different parameters) will fail with + * a {@link SnapshotCreationException} indicating the duplicate naming. + *

+ * Snapshot names follow the same naming constraints as tables in HBase. See + * {@link HTableDescriptor#isLegalTableName(byte[])}. + * @param snapshotName name of the snapshot to be created + * @param tableName name of the table for which snapshot is created + * @param flushType if the snapshot should be taken without flush memstore first + * @throws IOException if a remote or network exception occurs + * @throws SnapshotCreationException if snapshot creation failed + * @throws IllegalArgumentException if the snapshot request is formatted incorrectly + */ + public void snapshot(final byte[] snapshotName, final byte[] tableName, + final SnapshotDescription.Type flushType) throws + IOException, SnapshotCreationException, IllegalArgumentException { + snapshot(Bytes.toString(snapshotName), Bytes.toString(tableName), flushType); + } + /** public void snapshot(final String snapshotName, * Create a timestamp consistent snapshot for the given table. final byte[] tableName) throws IOException, diff --git a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/HBaseProtos.java b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/HBaseProtos.java index 238db31..28b3f8d 100644 --- a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/HBaseProtos.java +++ b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/HBaseProtos.java @@ -10476,6 +10476,7 @@ public final class HBaseProtos { * FLUSH = 1; */ FLUSH(1, 1), + SKIPFLUSH(2, 2), ; /** @@ -10494,6 +10495,7 @@ public final class HBaseProtos { switch (value) { case 0: return DISABLED; case 1: return FLUSH; + case 2: return SKIPFLUSH; default: return null; } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/FlushSnapshotSubprocedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/FlushSnapshotSubprocedure.java index b0a1b33..468c1af 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/FlushSnapshotSubprocedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/FlushSnapshotSubprocedure.java @@ -48,6 +48,7 @@ public class FlushSnapshotSubprocedure extends Subprocedure { private final List regions; private final SnapshotDescription snapshot; private final SnapshotSubprocedurePool taskManager; + private boolean snapshotSkipFlush = false; public FlushSnapshotSubprocedure(ProcedureMember member, ForeignExceptionDispatcher errorListener, long wakeFrequency, long timeout, @@ -55,6 +56,10 @@ public class FlushSnapshotSubprocedure extends Subprocedure { SnapshotSubprocedurePool taskManager) { super(member, snapshot.getName(), errorListener, wakeFrequency, timeout); this.snapshot = snapshot; + + if (this.snapshot.getType() == SnapshotDescription.Type.SKIPFLUSH) { + snapshotSkipFlush = true; + } this.regions = regions; this.taskManager = taskManager; } @@ -78,10 +83,25 @@ public class FlushSnapshotSubprocedure extends Subprocedure { LOG.debug("Starting region operation on " + region); region.startRegionOperation(); try { - LOG.debug("Flush Snapshotting region " + region.toString() + " started..."); - region.flushcache(); + if (snapshotSkipFlush) { + /* + * This is to take an online-snapshot without force a coordinated flush to prevent pause + * The snapshot type is defined inside the snapshot description. FlushSnapshotSubprocedure + * should be renamed to distributedSnapshotSubprocedure, and the flush() behavior can be + * turned on/off based on the flush type. + * To minimized the code change, class name is not changed. + */ + LOG.debug("take snapshot without flush memstore first"); + } else { + LOG.debug("Flush Snapshotting region " + region.toString() + " started..."); + region.flushcache(); + } region.addRegionToSnapshot(snapshot, monitor); - LOG.debug("... Flush Snapshotting region " + region.toString() + " completed."); + if (snapshotSkipFlush) { + LOG.debug("... SkipFlush Snapshotting region " + region.toString() + " completed."); + } else { + LOG.debug("... Flush Snapshotting region " + region.toString() + " completed."); + } } finally { LOG.debug("Closing region operation on " + region); region.closeRegionOperation(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/RegionServerSnapshotManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/RegionServerSnapshotManager.java index 4a4ee79..e78d690 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/RegionServerSnapshotManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/RegionServerSnapshotManager.java @@ -186,6 +186,19 @@ public class RegionServerSnapshotManager extends RegionServerProcedureManager { new SnapshotSubprocedurePool(rss.getServerName().toString(), conf); return new FlushSnapshotSubprocedure(member, exnDispatcher, wakeMillis, timeoutMillis, involvedRegions, snapshot, taskManager); + case SKIPFLUSH: + /* + * This is to take an online-snapshot without force a coordinated flush to prevent pause + * The snapshot type is defined inside the snapshot description. FlushSnapshotSubprocedure + * should be renamed to distributedSnapshotSubprocedure, and the flush() behavior can be + * turned on/off based on the flush type. + * To minimized the code change, class name is not changed. + */ + SnapshotSubprocedurePool taskManager2 = + new SnapshotSubprocedurePool(rss.getServerName().toString(), conf); + return new FlushSnapshotSubprocedure(member, exnDispatcher, wakeMillis, + timeoutMillis, involvedRegions, snapshot, taskManager2); + default: throw new UnsupportedOperationException("Unrecognized snapshot type:" + snapshot.getType()); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java index ee12d91..72a6b51 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java @@ -187,6 +187,58 @@ public class TestFlushSnapshotFromClient { admin, fs, false, new Path(rootDir, HConstants.HREGION_LOGDIR_NAME), snapshotServers); } + /** + * Test snapshotting a table that is online without flushing + * @throws Exception + */ + @Test + public void testSkipFlushTableSnapshot() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + // make sure we don't fail on listing snapshots + SnapshotTestingUtils.assertNoSnapshots(admin); + + // put some stuff in the table + HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME); + UTIL.loadTable(table, TEST_FAM); + + // get the name of all the regionservers hosting the snapshotted table + Set snapshotServers = new HashSet(); + List servers = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads(); + for (RegionServerThread server : servers) { + if (server.getRegionServer().getOnlineRegions(TABLE_NAME).size() > 0) { + snapshotServers.add(server.getRegionServer().getServerName().toString()); + } + } + + LOG.debug("FS state before snapshot:"); + FSUtils.logFileSystemState(UTIL.getTestFileSystem(), + FSUtils.getRootDir(UTIL.getConfiguration()), LOG); + + // take a snapshot of the enabled table + String snapshotString = "skipFlushTableSnapshot"; + byte[] snapshot = Bytes.toBytes(snapshotString); + admin.snapshot(snapshotString, STRING_TABLE_NAME, SnapshotDescription.Type.SKIPFLUSH); + LOG.debug("Snapshot completed."); + + // make sure we have the snapshot + List snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, + snapshot, TABLE_NAME); + + // make sure its a valid snapshot + FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem(); + Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir(); + LOG.debug("FS state after snapshot:"); + FSUtils.logFileSystemState(UTIL.getTestFileSystem(), + FSUtils.getRootDir(UTIL.getConfiguration()), LOG); + + SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir, + admin, fs, false, new Path(rootDir, HConstants.HREGION_LOGDIR_NAME), snapshotServers); + + admin.deleteSnapshot(snapshot); + snapshots = admin.listSnapshots(); + SnapshotTestingUtils.assertNoSnapshots(admin); + } + /** * Test simple flush snapshotting a table that is online diff --git a/hbase-shell/src/main/ruby/hbase.rb b/hbase-shell/src/main/ruby/hbase.rb index 3c09c4d..6cf154b 100644 --- a/hbase-shell/src/main/ruby/hbase.rb +++ b/hbase-shell/src/main/ruby/hbase.rb @@ -61,7 +61,8 @@ module HBaseConstants ATTRIBUTES="ATTRIBUTES" VISIBILITY="VISIBILITY" AUTHORIZATIONS = "AUTHORIZATIONS" - + SKIP_FLUSH = 'SKIP_FLUSH' + # Load constants from hbase java API def self.promote_constants(constants) # The constants to import are all in uppercase diff --git a/hbase-shell/src/main/ruby/hbase/admin.rb b/hbase-shell/src/main/ruby/hbase/admin.rb index 5cb7903..1d62bd8 100644 --- a/hbase-shell/src/main/ruby/hbase/admin.rb +++ b/hbase-shell/src/main/ruby/hbase/admin.rb @@ -22,6 +22,7 @@ java_import java.util.Arrays java_import org.apache.hadoop.hbase.util.Pair java_import org.apache.hadoop.hbase.util.RegionSplitter java_import org.apache.hadoop.hbase.util.Bytes +java_import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos::SnapshotDescription # Wrapper for org.apache.hadoop.hbase.client.HBaseAdmin @@ -691,8 +692,18 @@ module Hbase #---------------------------------------------------------------------------------------------- # Take a snapshot of specified table - def snapshot(table, snapshot_name) - @admin.snapshot(snapshot_name.to_java_bytes, table.to_java_bytes) + def snapshot(table, snapshot_name, *args) + if args.empty? + @admin.snapshot(snapshot_name.to_java_bytes, table.to_java_bytes) + else + args.each do |arg| + if arg[SKIP_FLUSH] = 'true' + @admin.snapshot(snapshot_name.to_java_bytes, table.to_java_bytes, SnapshotDescription::Type::SKIPFLUSH) + else + @admin.snapshot(snapshot_name.to_java_bytes, table.to_java_bytes) + end + end + end end #---------------------------------------------------------------------------------------------- diff --git a/hbase-shell/src/main/ruby/shell/commands/snapshot.rb b/hbase-shell/src/main/ruby/shell/commands/snapshot.rb index 62de845..e27c133 100644 --- a/hbase-shell/src/main/ruby/shell/commands/snapshot.rb +++ b/hbase-shell/src/main/ruby/shell/commands/snapshot.rb @@ -24,13 +24,13 @@ module Shell Take a snapshot of specified table. Examples: hbase> snapshot 'sourceTable', 'snapshotName' - hbase> snapshot 'namespace:sourceTable', 'snapshotName' + hbase> snapshot 'namespace:sourceTable', 'snapshotName', {SKIP_FLUSH => 'true'} EOF end - def command(table, snapshot_name) + def command(table, snapshot_name, *args) format_simple_command do - admin.snapshot(table, snapshot_name) + admin.snapshot(table, snapshot_name, *args) end end end