From 4072e3942203ebd9e5054559dfc071c1d6f45516 Mon Sep 17 00:00:00 2001 From: Guangxu Cheng Date: Thu, 5 Apr 2018 00:14:22 +0800 Subject: [PATCH] HBASE-20243 [Shell] Add shell command to create a new table by cloning the existent table --- .../java/org/apache/hadoop/hbase/client/Admin.java | 11 ++++ .../org/apache/hadoop/hbase/client/AsyncAdmin.java | 10 ++++ .../hadoop/hbase/client/AsyncHBaseAdmin.java | 6 ++ .../org/apache/hadoop/hbase/client/HBaseAdmin.java | 15 +++++ .../hadoop/hbase/client/RawAsyncHBaseAdmin.java | 65 +++++++++++++++++++- .../apache/hadoop/hbase/HBaseTestingUtility.java | 13 ++++ .../org/apache/hadoop/hbase/client/TestAdmin1.java | 70 ++++++++++++++++++++++ .../hbase/client/TestAsyncTableAdminApi.java | 64 ++++++++++++++++++++ hbase-shell/src/main/ruby/hbase/admin.rb | 7 +++ hbase-shell/src/main/ruby/shell.rb | 1 + hbase-shell/src/main/ruby/shell/commands.rb | 2 + .../main/ruby/shell/commands/clone_table_schema.rb | 41 +++++++++++++ hbase-shell/src/test/ruby/hbase/admin_test.rb | 47 +++++++++++++++ 13 files changed, 351 insertions(+), 1 deletion(-) create mode 100644 hbase-shell/src/main/ruby/shell/commands/clone_table_schema.rb diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java index b8546fa453..7d97414651 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java @@ -2718,4 +2718,15 @@ public interface Admin extends Abortable, Closeable { * @return List of servers that are not cleared */ List clearDeadServers(final List servers) throws IOException; + + /** + * Create a new table by cloning the existent table schema. + * + * @param tableName name of the table to be cloned + * @param newTableName name of the new table where the table will be created + * @param preserveSplits True if the splits should be preserved + * @throws IOException if a remote or network exception occurs + */ + void cloneTableSchema(final TableName tableName, final TableName newTableName, + final boolean preserveSplits) throws IOException; } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java index 35cdd3fd86..8141e74de7 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java @@ -1230,4 +1230,14 @@ public interface AsyncAdmin { * @return CacheEvictionStats related to the eviction wrapped by a {@link CompletableFuture}. */ CompletableFuture clearBlockCache(final TableName tableName); + + /** + * Create a new table by cloning the existent table schema. + * + * @param tableName name of the table to be cloned + * @param newTableName name of the new table where the table will be created + * @param preserveSplits True if the splits should be preserved + */ + CompletableFuture cloneTableSchema(final TableName tableName, + final TableName newTableName, final boolean preserveSplits); } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java index 9b2390c319..5b22668c6b 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java @@ -746,4 +746,10 @@ class AsyncHBaseAdmin implements AsyncAdmin { public CompletableFuture clearBlockCache(TableName tableName) { return wrap(rawAdmin.clearBlockCache(tableName)); } + + @Override + public CompletableFuture cloneTableSchema(TableName tableName, TableName newTableName, + boolean preserveSplits) { + return wrap(rawAdmin.cloneTableSchema(tableName, newTableName, preserveSplits)); + } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java index 86859847be..172db5b647 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java @@ -4227,4 +4227,19 @@ public class HBaseAdmin implements Admin { } }); } + + @Override + public void cloneTableSchema(final TableName tableName, final TableName newTableName, + final boolean preserveSplits) throws IOException { + checkTableExists(tableName); + if (tableExists(newTableName)) { + throw new TableExistsException(newTableName); + } + TableDescriptor htd = TableDescriptorBuilder.copy(newTableName, getTableDescriptor(tableName)); + if (preserveSplits) { + createTable(htd, getTableSplits(tableName)); + } else { + createTable(htd); + } + } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java index 050bfe2229..0fd0e59ab9 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java @@ -414,7 +414,7 @@ class RawAsyncHBaseAdmin implements AsyncAdmin { } /** - * {@link #listTables(boolean)} + * {@link #listTableDescriptors(boolean)} */ @Override public CompletableFuture> listTableDescriptors(Pattern pattern, @@ -3468,6 +3468,69 @@ class RawAsyncHBaseAdmin implements AsyncAdmin { return future; } + @Override + public CompletableFuture cloneTableSchema(TableName tableName, TableName newTableName, + boolean preserveSplits) { + CompletableFuture future = new CompletableFuture<>(); + tableExists(tableName).whenComplete( + (exist, err) -> { + if (err != null) { + future.completeExceptionally(err); + return; + } + if (!exist) { + future.completeExceptionally(new TableNotFoundException(tableName)); + return; + } + tableExists(newTableName).whenComplete( + (exist1, err1) -> { + if (err1 != null) { + future.completeExceptionally(err1); + return; + } + if (exist1) { + future.completeExceptionally(new TableExistsException(newTableName)); + return; + } + getDescriptor(tableName).whenComplete( + (tableDesc, err2) -> { + if (err2 != null) { + future.completeExceptionally(err2); + return; + } + TableDescriptor newTableDesc + = TableDescriptorBuilder.copy(newTableName, tableDesc); + if (preserveSplits) { + getTableSplits(tableName).whenComplete((splits, err3) -> { + if (err3 != null) { + future.completeExceptionally(err3); + } else { + createTable(newTableDesc, splits).whenComplete( + (result, err4) -> { + if (err4 != null) { + future.completeExceptionally(err4); + } else { + future.complete(result); + } + }); + } + }); + } else { + createTable(newTableDesc).whenComplete( + (result, err5) -> { + if (err5 != null) { + future.completeExceptionally(err5); + } else { + future.complete(result); + } + }); + } + }); + }); + }); + return future; + } + private CompletableFuture clearBlockCache(ServerName serverName, List hris) { return this. newAdminCaller().action((controller, stub) -> this diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java index 3c3cb0a0f7..8faeb5d293 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java @@ -41,6 +41,7 @@ import java.util.Collection; import java.util.Collections; import java.util.EnumSet; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.NavigableSet; @@ -4138,4 +4139,16 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { } return numHFiles; } + + public void verifyTableDescriptorIgnoreTableName(TableDescriptor ltd, TableDescriptor rtd) { + assertEquals(ltd.getValues().hashCode(), rtd.getValues().hashCode()); + Collection ltdFamilies = Arrays.asList(ltd.getColumnFamilies()); + Collection rtdFamilies = Arrays.asList(rtd.getColumnFamilies()); + assertEquals(ltdFamilies.size(), rtdFamilies.size()); + for (Iterator it = ltdFamilies.iterator(), it2 = + rtdFamilies.iterator(); it.hasNext();) { + assertEquals(0, + ColumnFamilyDescriptor.COMPARATOR.compare(it.next(), it2.next())); + } + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java index 8ac2ddafcb..6e376a4e29 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java @@ -41,6 +41,7 @@ import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.InvalidFamilyOperationException; import org.apache.hadoop.hbase.MetaTableAccessor; import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableExistsException; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.TableNotDisabledException; import org.apache.hadoop.hbase.TableNotEnabledException; @@ -1421,4 +1422,73 @@ public class TestAdmin1 { this.admin.getConnection(), tableName, true); assertEquals(1, allRegions.size()); } + + @Test + public void testCloneTableSchema() throws Exception { + final TableName tableName = TableName.valueOf(name.getMethodName()); + final TableName newTableName = TableName.valueOf(tableName.getNameAsString() + "_new"); + testCloneTableSchema(tableName, newTableName, false); + } + + @Test + public void testCloneTableSchemaPreservingSplits() throws Exception { + final TableName tableName = TableName.valueOf(name.getMethodName()); + final TableName newTableName = TableName.valueOf(tableName.getNameAsString() + "_new"); + testCloneTableSchema(tableName, newTableName, true); + } + + private void testCloneTableSchema(final TableName tableName, + final TableName newTableName, boolean preserveSplits) throws Exception { + byte[] FAMILY_0 = Bytes.toBytes("cf0"); + byte[] FAMILY_1 = Bytes.toBytes("cf1"); + byte[][] splitKeys = new byte[2][]; + splitKeys[0] = Bytes.toBytes(4); + splitKeys[1] = Bytes.toBytes(8); + + // test for non-existent source table + try { + admin.cloneTableSchema(tableName, newTableName, preserveSplits); + fail("Should have failed to create a new table by cloning non-existent source table."); + } catch (TableNotFoundException ex) { + // expected + } + + // Create the table + TableDescriptorBuilder builder = TableDescriptorBuilder + .newBuilder(tableName) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY_0)) + .setColumnFamily(ColumnFamilyDescriptorBuilder + .newBuilder(FAMILY_1) + .setBlocksize(8 * 1024) + .setBlockCacheEnabled(false) + .setInMemory(true) + .build() + ); + admin.createTable(builder.build(), splitKeys); + + boolean tableAvailable = admin.isTableAvailable(tableName, splitKeys); + assertTrue("Table should be created with splitKyes + 1 rows in META", tableAvailable); + + // clone & Verify + admin.cloneTableSchema(tableName, newTableName, preserveSplits); + TableDescriptor tableDesc = admin.getDescriptor(tableName); + TableDescriptor newTableDesc = admin.getDescriptor(newTableName); + TEST_UTIL.verifyTableDescriptorIgnoreTableName(tableDesc, newTableDesc); + if (preserveSplits) { + assertEquals(3, TEST_UTIL.getHBaseCluster().getRegions(newTableName).size()); + boolean newTableAvailable = admin.isTableAvailable(newTableName, splitKeys); + assertTrue("New table should be created with splitKyes + 1 rows in META", + newTableAvailable); + } else { + assertEquals(1, TEST_UTIL.getHBaseCluster().getRegions(newTableName).size()); + } + + // test for existent destination table + try { + admin.cloneTableSchema(tableName, newTableName, preserveSplits); + fail("Should have failed to create a existent table."); + } catch (TableExistsException ex) { + // expected + } + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi.java index 2c948dd934..9fb5f0133f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi.java @@ -36,7 +36,9 @@ import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableExistsException; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.TableNotFoundException; import org.apache.hadoop.hbase.master.LoadBalancer; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.LargeTests; @@ -366,4 +368,66 @@ public class TestAsyncTableAdminApi extends TestAsyncAdminBase { assertEquals(1, TEST_UTIL.getHBaseCluster().getRegions(tableName).size()); } } + + @Test + public void testCloneTableSchema() throws Exception { + final TableName newTableName = TableName.valueOf(tableName.getNameAsString() + "_new"); + testCloneTableSchema(tableName, newTableName, false); + } + + @Test + public void testCloneTableSchemaPreservingSplits() throws Exception { + final TableName newTableName = TableName.valueOf(tableName.getNameAsString() + "_new"); + testCloneTableSchema(tableName, newTableName, true); + } + + private void testCloneTableSchema(final TableName tableName, + final TableName newTableName, boolean preserveSplits) throws Exception { + byte[][] splitKeys = new byte[2][]; + splitKeys[0] = Bytes.toBytes(4); + splitKeys[1] = Bytes.toBytes(8); + // test for non-existent source table + try { + admin.cloneTableSchema(tableName, newTableName, preserveSplits).join(); + fail("Should have failed when source table doesn't exist."); + } catch (CompletionException e) { + assertTrue(e.getCause() instanceof TableNotFoundException); + } + // Create the table + TableDescriptorBuilder builder = TableDescriptorBuilder + .newBuilder(tableName) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY_0)) + .setColumnFamily(ColumnFamilyDescriptorBuilder + .newBuilder(FAMILY_1) + .setBlocksize(8 * 1024) + .setBlockCacheEnabled(false) + .setInMemory(true) + .build() + ); + admin.createTable(builder.build(), splitKeys).join(); + + boolean tableAvailable = admin.isTableAvailable(tableName, splitKeys).get(); + assertTrue("Table should be created with splitKyes + 1 rows in META", tableAvailable); + + // clone & Verify + admin.cloneTableSchema(tableName, newTableName, preserveSplits).join(); + TableDescriptor tableDesc = admin.getDescriptor(tableName).get(); + TableDescriptor newTableDesc = admin.getDescriptor(newTableName).get(); + TEST_UTIL.verifyTableDescriptorIgnoreTableName(tableDesc, newTableDesc); + if (preserveSplits) { + assertEquals(3, TEST_UTIL.getHBaseCluster().getRegions(newTableName).size()); + boolean newTableAvailable = admin.isTableAvailable(newTableName, splitKeys).get(); + assertTrue("New table should be created with splitKyes + 1 rows in META", + newTableAvailable); + } else { + assertEquals(1, TEST_UTIL.getHBaseCluster().getRegions(newTableName).size()); + } + + try { + admin.cloneTableSchema(tableName, newTableName, preserveSplits).join(); + fail("Should have failed when destination table exists."); + } catch (CompletionException e) { + assertTrue(e.getCause() instanceof TableExistsException); + } + } } diff --git a/hbase-shell/src/main/ruby/hbase/admin.rb b/hbase-shell/src/main/ruby/hbase/admin.rb index e409e39b9b..e275c0e24e 100644 --- a/hbase-shell/src/main/ruby/hbase/admin.rb +++ b/hbase-shell/src/main/ruby/hbase/admin.rb @@ -1300,5 +1300,12 @@ module Hbase def list_liveservers @admin.getClusterStatus.getServers.to_a end + + #---------------------------------------------------------------------------------------------- + # create a new table by cloning the existent table schema. + def clone_table_schema(table_name, new_table_name, preserve_splits = true) + @admin.cloneTableSchema(TableName.valueOf(table_name), + TableName.valueOf(new_table_name), preserve_splits) + end end end diff --git a/hbase-shell/src/main/ruby/shell.rb b/hbase-shell/src/main/ruby/shell.rb index 5e563dfea5..b4b5d19820 100644 --- a/hbase-shell/src/main/ruby/shell.rb +++ b/hbase-shell/src/main/ruby/shell.rb @@ -286,6 +286,7 @@ Shell.load_command_group( get_table locate_region list_regions + clone_table_schema ], aliases: { 'describe' => ['desc'] diff --git a/hbase-shell/src/main/ruby/shell/commands.rb b/hbase-shell/src/main/ruby/shell/commands.rb index d7730cf891..0f036b54cd 100644 --- a/hbase-shell/src/main/ruby/shell/commands.rb +++ b/hbase-shell/src/main/ruby/shell/commands.rb @@ -136,6 +136,8 @@ module Shell end end if cause.is_a?(org.apache.hadoop.hbase.TableExistsException) + strs = cause.to_s.split(' ') + raise "Table already exists: #{strs[0]}!" if strs.size == 1 raise "Table already exists: #{args.first}!" end # To be safe, here only AccessDeniedException is considered. In future diff --git a/hbase-shell/src/main/ruby/shell/commands/clone_table_schema.rb b/hbase-shell/src/main/ruby/shell/commands/clone_table_schema.rb new file mode 100644 index 0000000000..cc962eed79 --- /dev/null +++ b/hbase-shell/src/main/ruby/shell/commands/clone_table_schema.rb @@ -0,0 +1,41 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class CloneTableSchema < Command + def help + <<-EOF +Create a new table by cloning the existent table schema. +There're no copies of data involved. +Just copy the table descriptor and split keys. + +Passing 'false' as the optional third parameter will +not preserve split keys. +Examples: + hbase> clone_table_schema 'table_name', 'new_table_name' + hbase> clone_table_schema 'table_name', 'new_table_name', false +EOF + end + + def command(table_name, new_table_name, preserve_splits = true) + admin.clone_table_schema(table_name, new_table_name, preserve_splits) + end + end + end +end diff --git a/hbase-shell/src/test/ruby/hbase/admin_test.rb b/hbase-shell/src/test/ruby/hbase/admin_test.rb index 929484c51d..637f616197 100644 --- a/hbase-shell/src/test/ruby/hbase/admin_test.rb +++ b/hbase-shell/src/test/ruby/hbase/admin_test.rb @@ -319,6 +319,53 @@ module Hbase admin.truncate_preserve(@create_test_name, $TEST_CLUSTER.getConfiguration) assert_equal(splits, table(@create_test_name)._get_splits_internal()) end + + define_test "clone_table_schema should create a new table by cloning the existent table schema." do + drop_test_table(@create_test_name) + new_table = "test_clone_table_schema_table" + drop_test_table(new_table) + command(:create, @create_test_name, + { NAME => 'a', + CACHE_BLOOMS_ON_WRITE => 'TRUE', + CACHE_INDEX_ON_WRITE => 'TRUE', + EVICT_BLOCKS_ON_CLOSE => 'TRUE', + COMPRESSION_COMPACT => 'GZ'}) + command(:clone_table_schema, @create_test_name, new_table, false) + assert_equal(['a:'], table(@create_test_name).get_all_columns.sort) + assert_match(/CACHE_BLOOMS_ON_WRITE/, admin.describe(new_table)) + assert_match(/CACHE_INDEX_ON_WRITE/, admin.describe(new_table)) + assert_match(/EVICT_BLOCKS_ON_CLOSE/, admin.describe(new_table)) + assert_match(/GZ/, admin.describe(new_table)) + end + + define_test "clone_table_schema should maintain the source table's region boundaries when + preserve_splits set to true" do + drop_test_table(@create_test_name) + new_table = "test_clone_table_schema_table" + drop_test_table(new_table) + command(:create, @create_test_name,'a',{NUMREGIONS => 10, SPLITALGO => 'HexStringSplit'}) + splits = table(@create_test_name)._get_splits_internal() + command(:clone_table_schema, @create_test_name, new_table, true) + assert_equal(splits, table(new_table)._get_splits_internal()) + end + + define_test "clone_table_schema should have failed when source table doesn't exist." do + drop_test_table(@create_test_name) + exception = assert_raise(RuntimeError) do + command(:clone_table_schema, @create_test_name, 'new_table') + end + assert_equal("Unknown table hbase_create_table_test_table!", exception.message) + end + + define_test "clone_table_schema should have failed when destination table exists." do + drop_test_table(@create_test_name) + command(:create, @create_test_name,'a') + command(:create, 'new_table','a') + exception = assert_raise(RuntimeError) do + command(:clone_table_schema, @create_test_name, 'new_table') + end + assert_equal("Table already exists: new_table!", exception.message) + end end # Simple administration methods tests -- 2.13.0.windows.1