From ac43b7e8af287018a0f4d0dcc9b04b60be9e8f48 Mon Sep 17 00:00:00 2001 From: TAK LON WU Date: Fri, 17 Aug 2018 15:55:10 -0700 Subject: [PATCH] HBASE-21318 Make RefreshHFilesClient runnable --- .../hbase/client/example/RefreshHFilesClient.java | 35 +++++++- .../client/example/TestRefreshHFilesClient.java | 47 +++++++++++ .../coprocessor/example/TestRefreshHFilesBase.java | 96 ++++++++++++++++++++++ .../example/TestRefreshHFilesEndpoint.java | 95 ++++----------------- 4 files changed, 189 insertions(+), 84 deletions(-) create mode 100644 hbase-examples/src/test/java/org/apache/hadoop/hbase/client/example/TestRefreshHFilesClient.java create mode 100644 hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestRefreshHFilesBase.java diff --git a/hbase-examples/src/main/java/org/apache/hadoop/hbase/client/example/RefreshHFilesClient.java b/hbase-examples/src/main/java/org/apache/hadoop/hbase/client/example/RefreshHFilesClient.java index 38f0362..4334c0c 100644 --- a/hbase-examples/src/main/java/org/apache/hadoop/hbase/client/example/RefreshHFilesClient.java +++ b/hbase-examples/src/main/java/org/apache/hadoop/hbase/client/example/RefreshHFilesClient.java @@ -19,9 +19,13 @@ package org.apache.hadoop.hbase.client.example; +import java.io.Closeable; +import java.io.IOException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Connection; @@ -31,15 +35,14 @@ import org.apache.hadoop.hbase.client.coprocessor.Batch; import org.apache.hadoop.hbase.ipc.BlockingRpcCallback; import org.apache.hadoop.hbase.ipc.ServerRpcController; import org.apache.hadoop.hbase.protobuf.generated.RefreshHFilesProtos; - -import java.io.Closeable; -import java.io.IOException; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; /** * This client class is for invoking the refresh HFile function deployed on the * Region Server side via the RefreshHFilesService. */ -public class RefreshHFilesClient implements Closeable { +public class RefreshHFilesClient extends Configured implements Tool, Closeable { private static final Log LOG = LogFactory.getLog(RefreshHFilesClient.class); private final Connection connection; @@ -92,4 +95,28 @@ public class RefreshHFilesClient implements Closeable { }); LOG.debug("Done refreshing HFiles"); } + + @Override + public int run(String[] args) throws Exception { + if (args.length != 1) { + String message = "When there are multiple HBase clusters sharing a common root directory, " + + "especially for read replica cluster (see detail in HBASE-18477), please consider to " + + "use this tool manually sync the flushed HFiles from the source cluster."; + message += "\nUsage: " + this.getClass().getName() + " tableName"; + System.out.println(message); + return -1; + } + final TableName tableName = TableName.valueOf(args[0]); + try { + refreshHFiles(tableName); + } catch (Throwable t) { + LOG.error("Refresh HFiles from table " + tableName.getNameAsString() + " failed: ", t); + return -1; + } + return 0; + } + + public static void main(String[] args) throws Exception { + ToolRunner.run(new RefreshHFilesClient(HBaseConfiguration.create()), args); + } } diff --git a/hbase-examples/src/test/java/org/apache/hadoop/hbase/client/example/TestRefreshHFilesClient.java b/hbase-examples/src/test/java/org/apache/hadoop/hbase/client/example/TestRefreshHFilesClient.java new file mode 100644 index 0000000..d5add30 --- /dev/null +++ b/hbase-examples/src/test/java/org/apache/hadoop/hbase/client/example/TestRefreshHFilesClient.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client.example; + +import static org.junit.Assert.assertEquals; + +import org.apache.hadoop.hbase.coprocessor.example.TestRefreshHFilesBase; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.testclassification.ClientTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.util.ToolRunner; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category({ ClientTests.class, MediumTests.class }) +public class TestRefreshHFilesClient extends TestRefreshHFilesBase { + + @BeforeClass + public static void setUp() { + setUp(HRegion.class.getName()); + } + + @Test + public void testRefreshHFilesClient() throws Exception { + addHFilesToRegions(); + assertEquals(2, HTU.getNumHFiles(TABLE_NAME, FAMILY)); + RefreshHFilesClient tool = new RefreshHFilesClient(HTU.getConfiguration()); + assertEquals(0, ToolRunner.run(tool, new String[] { TABLE_NAME.getNameAsString() })); + assertEquals(4, HTU.getNumHFiles(TABLE_NAME, FAMILY)); + } +} diff --git a/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestRefreshHFilesBase.java b/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestRefreshHFilesBase.java new file mode 100644 index 0000000..96ae2f8 --- /dev/null +++ b/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestRefreshHFilesBase.java @@ -0,0 +1,96 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.coprocessor.example; + +import java.io.IOException; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.MiniHBaseCluster; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.regionserver.Region; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.HFileTestUtil; +import org.junit.After; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TestRefreshHFilesBase { + protected static final Logger LOG = LoggerFactory.getLogger(TestRefreshHFilesBase.class); + protected static final HBaseTestingUtility HTU = new HBaseTestingUtility(); + protected static final TableName TABLE_NAME = TableName.valueOf("testRefreshRegionHFilesEP"); + protected static final byte[] FAMILY = Bytes.toBytes("family"); + protected static final byte[] QUALIFIER = Bytes.toBytes("qualifier"); + private static final int NUM_ROWS = 5; + private static final int NUM_MASTER = 1; + private static final int NUM_RS = 2; + private static final byte[][] SPLIT_KEY = new byte[][] { Bytes.toBytes("30") }; + private static final String HFILE_NAME = "123abcdef"; + + protected static Configuration CONF = HTU.getConfiguration(); + private static MiniHBaseCluster cluster; + private static HTable table; + private static HTableDescriptor desc; + private static Admin hbaseAdmin; + + + public static void setUp(String regionImpl) { + try { + CONF.set(HConstants.REGION_IMPL, regionImpl); + CONF.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 2); + + CONF.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, RefreshHFilesEndpoint.class.getName()); + cluster = HTU.startMiniCluster(NUM_MASTER, NUM_RS); + + // Create table + desc = new HTableDescriptor(TABLE_NAME); + desc.addFamily(new HColumnDescriptor(FAMILY)); + hbaseAdmin = cluster.getMaster().getConnection().getAdmin(); + hbaseAdmin.createTable(desc, SPLIT_KEY); + table = new HTable(HTU.getConfiguration(), TABLE_NAME); + + // this will create 2 regions spread across slaves + HTU.loadNumericRows(table, FAMILY, 1, 20); + HTU.flush(TABLE_NAME); + } catch (Exception ex) { + LOG.error("Couldn't finish setup", ex); + } + } + + @After + public void tearDown() throws Exception { + HTU.shutdownMiniCluster(); + } + + protected void addHFilesToRegions() throws IOException { + Path tableDir = desc.getTableDir(HTU.getDefaultRootDirPath(), TABLE_NAME.toBytes()); + for (Region region : cluster.getRegions(TABLE_NAME)) { + Path regionDir = new Path(tableDir, region.getRegionInfo().getEncodedName()); + Path familyDir = new Path(regionDir, Bytes.toString(FAMILY)); + HFileTestUtil.createHFile(HTU.getConfiguration(), HTU.getTestFileSystem(), + new Path(familyDir, HFILE_NAME), FAMILY, QUALIFIER, Bytes.toBytes("50"), + Bytes.toBytes("60"), NUM_ROWS); + } + } +} diff --git a/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestRefreshHFilesEndpoint.java b/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestRefreshHFilesEndpoint.java index 9a4e293..a70eec9 100644 --- a/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestRefreshHFilesEndpoint.java +++ b/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestRefreshHFilesEndpoint.java @@ -15,104 +15,38 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.hadoop.hbase.coprocessor.example; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.MiniHBaseCluster; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.RetriesExhaustedException; import org.apache.hadoop.hbase.client.example.RefreshHFilesClient; -import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.HStore; -import org.apache.hadoop.hbase.regionserver.Region; import org.apache.hadoop.hbase.regionserver.RegionServerServices; import org.apache.hadoop.hbase.regionserver.Store; import org.apache.hadoop.hbase.testclassification.MediumTests; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.HFileTestUtil; import org.apache.hadoop.hbase.wal.WAL; -import org.junit.After; import org.junit.Test; import org.junit.experimental.categories.Category; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; - @Category(MediumTests.class) -public class TestRefreshHFilesEndpoint { - private static final Log LOG = LogFactory.getLog(TestRefreshHFilesEndpoint.class); - private static final HBaseTestingUtility HTU = new HBaseTestingUtility(); - private static final int NUM_MASTER = 1; - private static final int NUM_RS = 2; - private static final TableName TABLE_NAME = TableName.valueOf("testRefreshRegionHFilesEP"); - private static final byte[] FAMILY = Bytes.toBytes("family"); - private static final byte[] QUALIFIER = Bytes.toBytes("qualifier"); - private static final byte[][] SPLIT_KEY = new byte[][] { Bytes.toBytes("30") }; - private static final int NUM_ROWS = 5; - private static final String HFILE_NAME = "123abcdef"; - - private static Configuration CONF = HTU.getConfiguration(); - private static MiniHBaseCluster cluster; - private static HTableDescriptor desc; - private static Admin hbaseAdmin; - private static HTable table; - - public static void setUp(String regionImpl) { - try { - CONF.set(HConstants.REGION_IMPL, regionImpl); - CONF.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 2); - - CONF.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, RefreshHFilesEndpoint.class.getName()); - cluster = HTU.startMiniCluster(NUM_MASTER, NUM_RS); - - // Create table - desc = new HTableDescriptor(TABLE_NAME); - desc.addFamily(new HColumnDescriptor(FAMILY)); - hbaseAdmin = cluster.getMaster().getConnection().getAdmin(); - hbaseAdmin.createTable(desc, SPLIT_KEY); - table = new HTable(HTU.getConfiguration(), TABLE_NAME); - - // this will create 2 regions spread across slaves - HTU.loadNumericRows(table, FAMILY, 1, 20); - HTU.flush(TABLE_NAME); - } catch (Exception ex) { - LOG.error("Couldn't finish setup", ex); - } - } - - @After - public void tearDown() throws Exception { - HTU.shutdownMiniCluster(); - } +public class TestRefreshHFilesEndpoint extends TestRefreshHFilesBase { @Test public void testRefreshRegionHFilesEndpoint() throws Exception { setUp(HRegion.class.getName()); - Path tableDir = desc.getTableDir(HTU.getDefaultRootDirPath(), TABLE_NAME.toBytes()); - for (Region region : cluster.getRegions(TABLE_NAME)) { - Path regionDir = new Path(tableDir, region.getRegionInfo().getEncodedName()); - Path familyDir = new Path(regionDir, Bytes.toString(FAMILY)); - HFileTestUtil - .createHFile(HTU.getConfiguration(), HTU.getTestFileSystem(), new Path(familyDir, HFILE_NAME), FAMILY, - QUALIFIER, Bytes.toBytes("50"), Bytes.toBytes("60"), NUM_ROWS); - } + addHFilesToRegions(); assertEquals(2, HTU.getNumHFiles(TABLE_NAME, FAMILY)); callRefreshRegionHFilesEndPoint(); assertEquals(4, HTU.getNumHFiles(TABLE_NAME, FAMILY)); @@ -132,7 +66,7 @@ public class TestRefreshHFilesEndpoint { if (rex.getCause() instanceof IOException) throw new IOException(); } catch (Throwable ex) { - LOG.error(ex); + LOG.error(ex.toString(), ex); fail("Couldn't call the RefreshRegionHFilesEndpoint"); } } @@ -141,8 +75,8 @@ public class TestRefreshHFilesEndpoint { HStoreWithFaultyRefreshHFilesAPI store; public HRegionForRefreshHFilesEP(final Path tableDir, final WAL wal, final FileSystem fs, - final Configuration confParam, final HRegionInfo regionInfo, - final HTableDescriptor htd, final RegionServerServices rsServices) { + final Configuration confParam, final HRegionInfo regionInfo, + final HTableDescriptor htd, final RegionServerServices rsServices) { super(tableDir, wal, fs, confParam, regionInfo, htd, rsServices); } @@ -155,7 +89,8 @@ public class TestRefreshHFilesEndpoint { */ try { if (this.store == null) - store = new HStoreWithFaultyRefreshHFilesAPI(this, new HColumnDescriptor(FAMILY), this.conf); + store = new HStoreWithFaultyRefreshHFilesAPI(this, new HColumnDescriptor(FAMILY), + this.conf); list.add(store); } catch (IOException ioe) { LOG.info("Couldn't instantiate custom store implementation", ioe); @@ -167,8 +102,8 @@ public class TestRefreshHFilesEndpoint { } public static class HStoreWithFaultyRefreshHFilesAPI extends HStore { - public HStoreWithFaultyRefreshHFilesAPI(final HRegion region, final HColumnDescriptor family, - final Configuration confParam) throws IOException { + public HStoreWithFaultyRefreshHFilesAPI(final HRegion region, + final HColumnDescriptor family, final Configuration confParam) throws IOException { super(region, family, confParam); } -- 2.10.1 (Apple Git-78)