--- .../java/org/apache/hadoop/hbase/client/Admin.java | 8 ++ .../org/apache/hadoop/hbase/client/HBaseAdmin.java | 44 ++++++++++ .../hadoop/hbase/protobuf/RequestConverter.java | 12 +++ hbase-protocol/src/main/protobuf/Admin.proto | 10 +++ .../hadoop/hbase/regionserver/HRegionServer.java | 10 +++ .../hadoop/hbase/regionserver/RSRpcServices.java | 14 ++++ .../apache/hadoop/hbase/HBaseTestingUtility.java | 18 ++++ .../hadoop/hbase/TestRefreshRegionHFiles.java | 98 ++++++++++++++++++++++ .../hadoop/hbase/master/MockRegionServer.java | 9 ++ .../replication/regionserver/TestReplicator.java | 7 ++ hbase-shell/src/main/ruby/hbase/admin.rb | 6 ++ hbase-shell/src/main/ruby/shell.rb | 1 + .../src/main/ruby/shell/commands/refresh_hfiles.rb | 38 +++++++++ 13 files changed, 275 insertions(+) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/TestRefreshRegionHFiles.java create mode 100644 hbase-shell/src/main/ruby/shell/commands/refresh_hfiles.rb diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java index 5b3744a..f4591dc 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java @@ -725,6 +725,14 @@ public interface Admin extends Abortable, Closeable { boolean isBalancerEnabled() throws IOException; /** + * Refresh HFiles for the table + * + * @param tableName table to refresh HFiles for + * @throws IOException if a remote or network exception occurs + */ + void refreshHFiles(final TableName tableName) throws IOException; + + /** * Invoke region normalizer. Can NOT run for various reasons. Check logs. * * @return True if region normalizer ran, false otherwise. diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java index 2cbeb9a..4f99764 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java @@ -87,6 +87,7 @@ import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.FlushRegionRequest import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoRequest; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse.CompactionState; +import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.RefreshRegionHFilesRequest; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.RollWALWriterRequest; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.RollWALWriterResponse; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.StopServerRequest; @@ -2435,6 +2436,49 @@ public class HBaseAdmin implements Admin { } /** + * {@inheritDoc} + */ + @Override + public void refreshHFiles(final TableName tableName) throws IOException { + ZooKeeperWatcher zookeeper = null; + try { + checkTableExists(tableName); + zookeeper = new ZooKeeperWatcher(conf, ZK_IDENTIFIER_PREFIX + connection.toString(), + new ThrowableAbortable()); + List> pairs = + MetaTableAccessor.getTableRegionsAndLocations(zookeeper, connection, tableName); + for (Pair pair: pairs) { + if (pair.getFirst().isOffline()) continue; + if (pair.getSecond() == null) continue; + try { + refreshHFiles(pair.getSecond(), pair.getFirst()); + } catch (NotServingRegionException e) { + if (LOG.isDebugEnabled()) { + LOG.debug("Trying to refresh HFiles for " + pair.getFirst() + ": " + + StringUtils.stringifyException(e)); + } + } + } + } finally { + if (zookeeper != null) { + zookeeper.close(); + } + } + } + + private void refreshHFiles(final ServerName sn, final HRegionInfo hri) throws IOException { + HBaseRpcController controller = rpcControllerFactory.newController(); + AdminService.BlockingInterface admin = this.connection.getAdmin(sn); + RefreshRegionHFilesRequest request = + RequestConverter.buildRefreshRegionHFilesRequest(hri.getRegionName()); + try { + admin.refreshRegionHFiles(controller, request); + } catch (ServiceException se) { + throw ProtobufUtil.getRemoteException(se); + } + } + + /** * Invoke region normalizer. Can NOT run for various reasons. Check logs. * * @return True if region normalizer ran, false otherwise. diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/RequestConverter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/RequestConverter.java index 8163130..8a7fdad 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/RequestConverter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/RequestConverter.java @@ -56,6 +56,7 @@ import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetStoreFileReques import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.MergeRegionsRequest; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionRequest; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionRequest.RegionOpenInfo; +import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.RefreshRegionHFilesRequest; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.RollWALWriterRequest; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.SplitRegionRequest; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.StopServerRequest; @@ -1438,6 +1439,17 @@ public final class RequestConverter { } /** + * Creates a protocol buffer RefreshRegionHFilesRequest + * + * @return a RefreshRegionHFilesRequest + */ + public static RefreshRegionHFilesRequest buildRefreshRegionHFilesRequest(final byte[] + regionName) { + RegionSpecifier region = buildRegionSpecifier(RegionSpecifierType.REGION_NAME, regionName); + return RefreshRegionHFilesRequest.newBuilder().setRegion(region).build(); + } + + /** * @see {@link #buildGetClusterStatusRequest} */ private static final GetClusterStatusRequest GET_CLUSTER_STATUS_REQUEST = diff --git a/hbase-protocol/src/main/protobuf/Admin.proto b/hbase-protocol/src/main/protobuf/Admin.proto index a1905a4..48c153b 100644 --- a/hbase-protocol/src/main/protobuf/Admin.proto +++ b/hbase-protocol/src/main/protobuf/Admin.proto @@ -256,6 +256,13 @@ message UpdateConfigurationRequest { message UpdateConfigurationResponse { } +message RefreshRegionHFilesRequest { + required RegionSpecifier region = 1; +} + +message RefreshRegionHFilesResponse { +} + service AdminService { rpc GetRegionInfo(GetRegionInfoRequest) returns(GetRegionInfoResponse); @@ -307,4 +314,7 @@ service AdminService { rpc UpdateConfiguration(UpdateConfigurationRequest) returns(UpdateConfigurationResponse); + + rpc RefreshRegionHFiles(RefreshRegionHFilesRequest) + returns(RefreshRegionHFilesResponse); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java index 107e1c3..a51f5bf 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java @@ -3478,6 +3478,16 @@ public class HRegionServer extends HasThread implements configurationManager.notifyAllObservers(conf); } + public void refreshRegionHFiles(Region region) { + try { + for (Store store : region.getStores()) { + store.refreshStoreFiles(); + } + } catch(IOException ioe) { + LOG.warn("Exception while trying to refresh store files: ", ioe); + } + } + @Override public HeapMemoryManager getHeapMemoryManager() { return hMemManager; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java index 4071fed..a4a8f34 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java @@ -128,6 +128,8 @@ import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionRequest; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionRequest.RegionOpenInfo; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionResponse; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionResponse.RegionOpeningState; +import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.RefreshRegionHFilesRequest; +import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.RefreshRegionHFilesResponse; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ReplicateWALEntryRequest; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ReplicateWALEntryResponse; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.RollWALWriterRequest; @@ -3131,4 +3133,16 @@ public class RSRpcServices implements HBaseRPCErrorHandler, } return UpdateConfigurationResponse.getDefaultInstance(); } + + @Override + public RefreshRegionHFilesResponse refreshRegionHFiles(RpcController controller, + RefreshRegionHFilesRequest request) + throws ServiceException { + try { + this.regionServer.refreshRegionHFiles(getRegion(request.getRegion())); + } catch (Exception e) { + throw new ServiceException(e); + } + return RefreshRegionHFilesResponse.newBuilder().build(); + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java index f60be66..1070501 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java @@ -4337,4 +4337,22 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { HBaseKerberosUtils.setKeytabFileForTesting(keytabFile.getAbsolutePath()); return kdc; } + + public int getNumHFiles(final TableName tableName, final byte[] family) { + int numHFiles = 0; + for (RegionServerThread regionServerThread : getMiniHBaseCluster().getRegionServerThreads()) { + numHFiles+= getNumHFilesForRS(regionServerThread.getRegionServer(), tableName, + family); + } + return numHFiles; + } + + public int getNumHFilesForRS(final HRegionServer rs, final TableName tableName, + final byte[] family) { + int numHFiles = 0; + for (Region region : rs.getOnlineRegions(tableName)) { + numHFiles += region.getStore(family).getStorefilesCount(); + } + return numHFiles; + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRefreshRegionHFiles.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRefreshRegionHFiles.java new file mode 100644 index 0000000..975be53 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRefreshRegionHFiles.java @@ -0,0 +1,98 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.regionserver.Region; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.HFileTestUtil; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static org.junit.Assert.assertEquals; + +@Category(MediumTests.class) +public class TestRefreshRegionHFiles { + private static final HBaseTestingUtility HTU = new HBaseTestingUtility(); + private static final int NUM_MASTER = 1; + private static final int NUM_RS = 2; + private static final TableName TABLE_NAME = TableName.valueOf("testRefreshRegionHFiles"); + private static final byte[] FAMILY = Bytes.toBytes("family"); + private static final byte[] QUALIFIER = Bytes.toBytes("qualifier"); + private static final byte[][] SPLIT_KEY = new byte[][] { Bytes.toBytes("10") }; + private static final int NUM_ROWS = 5; + private static final String HFILE_NAME = "123abcdef"; + + private static MiniHBaseCluster cluster; + private static HRegionServer rs1, rs2; + private static HTableDescriptor desc; + private static Admin hbaseAdmin; + private static HTable table; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + cluster = HTU.startMiniCluster(NUM_MASTER, NUM_RS); + rs1 = cluster.getRegionServer(0); + rs2 = cluster.getRegionServer(1); + + // Create table + desc = new HTableDescriptor(TABLE_NAME); + desc.addFamily(new HColumnDescriptor(FAMILY)); + hbaseAdmin = cluster.getMaster().getConnection().getAdmin(); + hbaseAdmin.createTable(desc, SPLIT_KEY); + table = new HTable(HTU.getConfiguration(), TABLE_NAME); + + // this will create 2 regions spread across slaves + HTU.loadNumericRows(table, FAMILY, 1, 20); + HTU.flush(TABLE_NAME); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + HTU.shutdownMiniCluster(); + } + + @Test + public void testRefreshRegionHFiles() throws Exception { + Path tableDir = desc.getTableDir(HTU.getDefaultRootDirPath(), TABLE_NAME.toBytes()); + for (Region region : cluster.getRegions(TABLE_NAME)) { + Path regionDir = new Path(tableDir, region.getRegionInfo().getEncodedName()); + Path familyDir = new Path(regionDir, Bytes.toString(FAMILY)); + HFileTestUtil + .createHFile(HTU.getConfiguration(), HTU.getTestFileSystem(), new Path(familyDir, HFILE_NAME), FAMILY, + QUALIFIER, Bytes.toBytes("50"), Bytes.toBytes("60"), NUM_ROWS); + } + assertEquals(2, HTU.getNumHFiles(TABLE_NAME, FAMILY)); + refreshRegionHFiles(rs1); + refreshRegionHFiles(rs2); + assertEquals(4, HTU.getNumHFiles(TABLE_NAME, FAMILY)); + } + + private void refreshRegionHFiles(HRegionServer rs) { + for (Region region : rs.getOnlineRegions(TABLE_NAME)) { + rs.refreshRegionHFiles(region); + } + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java index 14bd2fd..b87de29 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java @@ -70,6 +70,8 @@ import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.MergeRegionsReques import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.MergeRegionsResponse; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionRequest; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionResponse; +import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.RefreshRegionHFilesRequest; +import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.RefreshRegionHFilesResponse; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ReplicateWALEntryRequest; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ReplicateWALEntryResponse; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.RollWALWriterRequest; @@ -649,6 +651,13 @@ ClientProtos.ClientService.BlockingInterface, RegionServerServices { } @Override + public RefreshRegionHFilesResponse refreshRegionHFiles(RpcController controller, + RefreshRegionHFilesRequest request) + throws ServiceException { + return null; + } + + @Override public HeapMemoryManager getHeapMemoryManager() { return null; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicator.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicator.java index 6d15a1b..15efe31 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicator.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicator.java @@ -382,6 +382,13 @@ public class TestReplicator extends TestReplicationBase { UpdateConfigurationRequest request) throws ServiceException { return delegate.updateConfiguration(controller, request); } + + @Override + public RefreshRegionHFilesResponse refreshRegionHFiles(RpcController controller, + RefreshRegionHFilesRequest request) + throws ServiceException { + return delegate.refreshRegionHFiles(controller, request); + } } public class FailureInjectingReplicatorForTest extends ReplicatorForTest { diff --git a/hbase-shell/src/main/ruby/hbase/admin.rb b/hbase-shell/src/main/ruby/hbase/admin.rb index 502abe2..9cf5e75 100644 --- a/hbase-shell/src/main/ruby/hbase/admin.rb +++ b/hbase-shell/src/main/ruby/hbase/admin.rb @@ -170,6 +170,12 @@ module Hbase end #---------------------------------------------------------------------------------------------- + # Refresh HFiles for the table + def refresh_hfiles(table_name) + @admin.refreshHFiles(org.apache.hadoop.hbase.TableName.valueOf(table_name)) + end + + #---------------------------------------------------------------------------------------------- # Requests region normalization for all configured tables in the cluster # Returns true if normalizer ran successfully def normalize() diff --git a/hbase-shell/src/main/ruby/shell.rb b/hbase-shell/src/main/ruby/shell.rb index 99adf73..9370473 100644 --- a/hbase-shell/src/main/ruby/shell.rb +++ b/hbase-shell/src/main/ruby/shell.rb @@ -344,6 +344,7 @@ Shell.load_command_group( trace splitormerge_switch splitormerge_enabled + refresh_hfiles ], # TODO remove older hlog_roll command :aliases => { diff --git a/hbase-shell/src/main/ruby/shell/commands/refresh_hfiles.rb b/hbase-shell/src/main/ruby/shell/commands/refresh_hfiles.rb new file mode 100644 index 0000000..a361fde --- /dev/null +++ b/hbase-shell/src/main/ruby/shell/commands/refresh_hfiles.rb @@ -0,0 +1,38 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class RefreshHfiles < Command + def help + return <<-EOF +Refresh HFiles for the table. +For example: + + hbase> refresh_hfiles 'TABLENAME' + +EOF + end + def command(table_name) + format_simple_command do + admin.refresh_hfiles(table_name) + end + end + end + end +end -- 2.10.1 (Apple Git-78)