From 8e22290d9f613b5dabe8f0550adf78dd39bdc670 Mon Sep 17 00:00:00 2001 From: chenheng Date: Thu, 20 Aug 2015 11:59:06 +0800 Subject: [PATCH] HBASE-14227 Fold special cased MOB APIs into existing APIs --- .../java/org/apache/hadoop/hbase/client/Admin.java | 37 ++- .../org/apache/hadoop/hbase/client/HBaseAdmin.java | 306 ++++++++++----------- .../hbase/chaos/actions/CompactMobAction.java | 4 +- .../hbase/mob/compactions/TestMobCompactor.java | 12 +- hbase-shell/src/main/ruby/hbase/admin.rb | 8 +- 5 files changed, 176 insertions(+), 191 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java index f2fc958..65de0d4 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java @@ -1496,56 +1496,61 @@ public interface Admin extends Abortable, Closeable { public int getMasterInfoPort() throws IOException; /** - * Compact the mob files in all mob-enabled column families. Asynchronous operation. + * Compact a table. Asynchronous operation. * * @param tableName table to compact + * @param compactType * @throws IOException * @throws InterruptedException */ - void compactMobs(final TableName tableName) throws IOException, + void compact(final TableName tableName, CompactType compactType) throws IOException, InterruptedException; /** - * Compact the mob files in a mob-enabled column family. Asynchronous operation. + * Compact a column family within a table. Asynchronous operation. * * @param tableName table to compact * @param columnFamily column family within a table + * @param compactType * @throws IOException if not a mob column family or if a remote or network exception occurs * @throws InterruptedException */ - void compactMob(final TableName tableName, final byte[] columnFamily) throws IOException, - InterruptedException; + void compact(final TableName tableName, final byte[] columnFamily, CompactType compactType) + throws IOException, InterruptedException; /** - * Major compact the mob files in all mob-enabled column family. Asynchronous operation. + * Major compact a table. Asynchronous operation. * * @param tableName table to compact + * @param compactType * @throws IOException * @throws InterruptedException */ - void majorCompactMobs(final TableName tableName) throws IOException, + void majorCompact(final TableName tableName, CompactType compactType) throws IOException, InterruptedException; /** - * Major compact the mob files in a mob-enabled column family. Asynchronous operation. + * Major compact a column family within a table. Asynchronous operation. * * @param tableName table to compact * @param columnFamily column family within a table + * @param compactType * @throws IOException if not a mob column family or if a remote or network exception occurs * @throws InterruptedException */ - void majorCompactMob(final TableName tableName, final byte[] columnFamily) throws IOException, - InterruptedException; + void majorCompact(final TableName tableName, final byte[] columnFamily, CompactType compactType) + throws IOException, InterruptedException; /** * Get the current compaction state of a table. It could be in a compaction, or none. * * @param tableName table to examine + * @param compactType * @return the current compaction state * @throws IOException if a remote or network exception occurs */ - AdminProtos.GetRegionInfoResponse.CompactionState getMobCompactionState(final TableName tableName) - throws IOException; + AdminProtos.GetRegionInfoResponse.CompactionState getCompactionState(final TableName tableName, + CompactType compactType) throws IOException; /** * Return the set of supported security capabilities. @@ -1553,4 +1558,12 @@ public interface Admin extends Abortable, Closeable { * @throws UnsupportedOperationException */ List getSecurityCapabilities() throws IOException; + + public enum CompactType { + + NORMAL (0), + MOB (1); + + CompactType(int value) {} + } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java index 7ed303d..53784cf 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java @@ -1761,7 +1761,7 @@ public class HBaseAdmin implements Admin { @Override public void compact(final TableName tableName) throws IOException { - compact(tableName, null, false); + compact(tableName, null, false, CompactType.NORMAL); } /** @@ -1793,7 +1793,7 @@ public class HBaseAdmin implements Admin { try { compactRegion(tableNameOrRegionName, null, false); } catch (IllegalArgumentException e) { - compact(TableName.valueOf(tableNameOrRegionName), null, false); + compact(TableName.valueOf(tableNameOrRegionName), null, false, CompactType.NORMAL); } } @@ -1803,7 +1803,7 @@ public class HBaseAdmin implements Admin { @Override public void compact(final TableName tableName, final byte[] columnFamily) throws IOException { - compact(tableName, columnFamily, false); + compact(tableName, columnFamily, false, CompactType.NORMAL); } /** @@ -1836,7 +1836,7 @@ public class HBaseAdmin implements Admin { compactRegion(tableNameOrRegionName, columnFamily, false); } catch (IllegalArgumentException e) { // Bad region, try table - compact(TableName.valueOf(tableNameOrRegionName), columnFamily, false); + compact(TableName.valueOf(tableNameOrRegionName), columnFamily, false, CompactType.NORMAL); } } @@ -1857,7 +1857,7 @@ public class HBaseAdmin implements Admin { @Override public void majorCompact(final TableName tableName) throws IOException { - compact(tableName, null, true); + compact(tableName, null, true, CompactType.NORMAL); } /** @@ -1890,7 +1890,7 @@ public class HBaseAdmin implements Admin { compactRegion(tableNameOrRegionName, null, true); } catch (IllegalArgumentException e) { // Invalid region, try table - compact(TableName.valueOf(tableNameOrRegionName), null, true); + compact(TableName.valueOf(tableNameOrRegionName), null, true, CompactType.NORMAL); } } @@ -1900,7 +1900,7 @@ public class HBaseAdmin implements Admin { @Override public void majorCompact(final TableName tableName, final byte[] columnFamily) throws IOException { - compact(tableName, columnFamily, true); + compact(tableName, columnFamily, true, CompactType.NORMAL); } /** @@ -1933,7 +1933,7 @@ public class HBaseAdmin implements Admin { compactRegion(tableNameOrRegionName, columnFamily, true); } catch (IllegalArgumentException e) { // Invalid region, try table - compact(TableName.valueOf(tableNameOrRegionName), columnFamily, true); + compact(TableName.valueOf(tableNameOrRegionName), columnFamily, true, CompactType.NORMAL); } } @@ -1947,36 +1947,44 @@ public class HBaseAdmin implements Admin { * @throws IOException if a remote or network exception occurs * @throws InterruptedException */ - private void compact(final TableName tableName, final byte[] columnFamily,final boolean major) - throws IOException { - ZooKeeperWatcher zookeeper = null; - try { - checkTableExists(tableName); - zookeeper = new ZooKeeperWatcher(conf, ZK_IDENTIFIER_PREFIX + connection.toString(), - new ThrowableAbortable()); - List> pairs; - if (TableName.META_TABLE_NAME.equals(tableName)) { - pairs = new MetaTableLocator().getMetaRegionsAndLocations(zookeeper); - } else { - pairs = MetaTableAccessor.getTableRegionsAndLocations(connection, tableName); - } - for (Pair pair: pairs) { - if (pair.getFirst().isOffline()) continue; - if (pair.getSecond() == null) continue; + private void compact(final TableName tableName, final byte[] columnFamily,final boolean major, + CompactType compactType) throws IOException { + switch (compactType) { + case MOB: + ServerName master = getClusterStatus().getMaster(); + compact(master, getMobRegionInfo(tableName), major, columnFamily); + break; + case NORMAL: + ZooKeeperWatcher zookeeper = null; try { - compact(pair.getSecond(), pair.getFirst(), major, columnFamily); - } catch (NotServingRegionException e) { - if (LOG.isDebugEnabled()) { - LOG.debug("Trying to" + (major ? " major" : "") + " compact " + - pair.getFirst() + ": " + - StringUtils.stringifyException(e)); + checkTableExists(tableName); + zookeeper = new ZooKeeperWatcher(conf, ZK_IDENTIFIER_PREFIX + connection.toString(), + new ThrowableAbortable()); + List> pairs; + if (TableName.META_TABLE_NAME.equals(tableName)) { + pairs = new MetaTableLocator().getMetaRegionsAndLocations(zookeeper); + } else { + pairs = MetaTableAccessor.getTableRegionsAndLocations(connection, tableName); + } + for (Pair pair: pairs) { + if (pair.getFirst().isOffline()) continue; + if (pair.getSecond() == null) continue; + try { + compact(pair.getSecond(), pair.getFirst(), major, columnFamily); + } catch (NotServingRegionException e) { + if (LOG.isDebugEnabled()) { + LOG.debug("Trying to" + (major ? " major" : "") + " compact " + + pair.getFirst() + ": " + + StringUtils.stringifyException(e)); + } + } + } + } finally { + if (zookeeper != null) { + zookeeper.close(); } } - } - } finally { - if (zookeeper != null) { - zookeeper.close(); - } + break; } } @@ -2983,68 +2991,7 @@ public class HBaseAdmin implements Admin { @Override public CompactionState getCompactionState(final TableName tableName) throws IOException { - CompactionState state = CompactionState.NONE; - ZooKeeperWatcher zookeeper = - new ZooKeeperWatcher(conf, ZK_IDENTIFIER_PREFIX + connection.toString(), - new ThrowableAbortable()); - try { - checkTableExists(tableName); - List> pairs; - if (TableName.META_TABLE_NAME.equals(tableName)) { - pairs = new MetaTableLocator().getMetaRegionsAndLocations(zookeeper); - } else { - pairs = MetaTableAccessor.getTableRegionsAndLocations(connection, tableName); - } - for (Pair pair: pairs) { - if (pair.getFirst().isOffline()) continue; - if (pair.getSecond() == null) continue; - try { - ServerName sn = pair.getSecond(); - AdminService.BlockingInterface admin = this.connection.getAdmin(sn); - GetRegionInfoRequest request = RequestConverter.buildGetRegionInfoRequest( - pair.getFirst().getRegionName(), true); - GetRegionInfoResponse response = admin.getRegionInfo(null, request); - switch (response.getCompactionState()) { - case MAJOR_AND_MINOR: - return CompactionState.MAJOR_AND_MINOR; - case MAJOR: - if (state == CompactionState.MINOR) { - return CompactionState.MAJOR_AND_MINOR; - } - state = CompactionState.MAJOR; - break; - case MINOR: - if (state == CompactionState.MAJOR) { - return CompactionState.MAJOR_AND_MINOR; - } - state = CompactionState.MINOR; - break; - case NONE: - default: // nothing, continue - } - } catch (NotServingRegionException e) { - if (LOG.isDebugEnabled()) { - LOG.debug("Trying to get compaction state of " + - pair.getFirst() + ": " + - StringUtils.stringifyException(e)); - } - } catch (RemoteException e) { - if (e.getMessage().indexOf(NotServingRegionException.class.getName()) >= 0) { - if (LOG.isDebugEnabled()) { - LOG.debug("Trying to get compaction state of " + pair.getFirst() + ": " - + StringUtils.stringifyException(e)); - } - } else { - throw e; - } - } - } - } catch (ServiceException se) { - throw ProtobufUtil.getRemoteException(se); - } finally { - zookeeper.close(); - } - return state; + return getCompactionState(tableName, CompactType.NORMAL); } /** @@ -4138,102 +4085,121 @@ public class HBaseAdmin implements Admin { * {@inheritDoc} */ @Override - public void compactMob(final TableName tableName, final byte[] columnFamily) + public void compact(final TableName tableName, final byte[] columnFamily, CompactType compactType) throws IOException, InterruptedException { - checkTableNameNotNull(tableName); - checkFamilyNameNotNull(columnFamily); - validateMobColumnFamily(tableName, columnFamily); - compactMob(tableName, columnFamily, false); + compact(tableName, columnFamily, false, compactType); } /** * {@inheritDoc} */ @Override - public void compactMobs(final TableName tableName) throws IOException, InterruptedException { - checkTableNameNotNull(tableName); - compactMob(tableName, null, false); + public void compact(final TableName tableName, CompactType compactType) throws IOException, InterruptedException { + compact(tableName, null, false, compactType); } /** * {@inheritDoc} */ @Override - public void majorCompactMob(final TableName tableName, final byte[] columnFamily) - throws IOException, InterruptedException { - checkTableNameNotNull(tableName); - checkFamilyNameNotNull(columnFamily); - validateMobColumnFamily(tableName, columnFamily); - compactMob(tableName, columnFamily, true); + public void majorCompact(final TableName tableName, final byte[] columnFamily, CompactType compactType) + throws IOException, InterruptedException { + compact(tableName, columnFamily, true, compactType); } /** * {@inheritDoc} */ @Override - public void majorCompactMobs(final TableName tableName) throws IOException, InterruptedException { - checkTableNameNotNull(tableName); - compactMob(tableName, null, true); + public void majorCompact(final TableName tableName, CompactType compactType) + throws IOException, InterruptedException { + compact(tableName, null, true, compactType); } /** * {@inheritDoc} */ @Override - public CompactionState getMobCompactionState(TableName tableName) throws IOException { - checkTableNameNotNull(tableName); - try { - ServerName master = getClusterStatus().getMaster(); - HRegionInfo info = new HRegionInfo(tableName, Bytes.toBytes(".mob"), - HConstants.EMPTY_END_ROW, false, 0); - GetRegionInfoRequest request = RequestConverter.buildGetRegionInfoRequest( - info.getRegionName(), true); - GetRegionInfoResponse response = this.connection.getAdmin(master) - .getRegionInfo(null, request); - return response.getCompactionState(); - } catch (ServiceException se) { - throw ProtobufUtil.getRemoteException(se); - } - } - - /** - * Compacts the mob files in a mob-enabled column family. Asynchronous operation. - * @param tableName The table to compact. - * @param columnFamily The column family to compact. If it is null, all the mob-enabled - * column families in this table will be compacted. - * @param major Whether to select all the mob files in the compaction. - * @throws IOException - * @throws InterruptedException - */ - private void compactMob(final TableName tableName, final byte[] columnFamily, boolean major) - throws IOException, InterruptedException { - // get the mob region info, this is a dummy region. - HRegionInfo info = new HRegionInfo(tableName, Bytes.toBytes(".mob"), HConstants.EMPTY_END_ROW, - false, 0); - ServerName master = getClusterStatus().getMaster(); - compact(master, info, major, columnFamily); - } - - private void checkTableNameNotNull(TableName tableName) { - if (tableName == null) { - throw new IllegalArgumentException("TableName cannot be null"); - } - } - - private void checkFamilyNameNotNull(byte[] columnFamily) { - if (columnFamily == null) { - throw new IllegalArgumentException("The column family name cannot be null"); - } - } - - private void validateMobColumnFamily(TableName tableName, byte[] columnFamily) - throws IOException { - HTableDescriptor htd = getTableDescriptor(tableName); - HColumnDescriptor family = htd.getFamily(columnFamily); - if (family == null || !family.isMobEnabled()) { - throw new IllegalArgumentException("Column family " + Bytes.toString(columnFamily) - + " is not a mob column family"); + public CompactionState getCompactionState(TableName tableName, CompactType compactType) throws IOException { + CompactionState state = CompactionState.NONE; + checkTableExists(tableName); + switch (compactType) { + case MOB: + try { + ServerName master = getClusterStatus().getMaster(); + HRegionInfo info = getMobRegionInfo(tableName); + GetRegionInfoRequest request = RequestConverter.buildGetRegionInfoRequest( + info.getRegionName(), true); + GetRegionInfoResponse response = this.connection.getAdmin(master) + .getRegionInfo(null, request); + state = response.getCompactionState(); + } catch (ServiceException se) { + throw ProtobufUtil.getRemoteException(se); + } + break; + case NORMAL: + ZooKeeperWatcher zookeeper = + new ZooKeeperWatcher(conf, ZK_IDENTIFIER_PREFIX + connection.toString(), + new ThrowableAbortable()); + try { + List> pairs; + if (TableName.META_TABLE_NAME.equals(tableName)) { + pairs = new MetaTableLocator().getMetaRegionsAndLocations(zookeeper); + } else { + pairs = MetaTableAccessor.getTableRegionsAndLocations(connection, tableName); + } + for (Pair pair : pairs) { + if (pair.getFirst().isOffline()) continue; + if (pair.getSecond() == null) continue; + try { + ServerName sn = pair.getSecond(); + AdminService.BlockingInterface admin = this.connection.getAdmin(sn); + GetRegionInfoRequest request = RequestConverter.buildGetRegionInfoRequest( + pair.getFirst().getRegionName(), true); + GetRegionInfoResponse response = admin.getRegionInfo(null, request); + switch (response.getCompactionState()) { + case MAJOR_AND_MINOR: + return CompactionState.MAJOR_AND_MINOR; + case MAJOR: + if (state == CompactionState.MINOR) { + return CompactionState.MAJOR_AND_MINOR; + } + state = CompactionState.MAJOR; + break; + case MINOR: + if (state == CompactionState.MAJOR) { + return CompactionState.MAJOR_AND_MINOR; + } + state = CompactionState.MINOR; + break; + case NONE: + default: // nothing, continue + } + } catch (NotServingRegionException e) { + if (LOG.isDebugEnabled()) { + LOG.debug("Trying to get compaction state of " + + pair.getFirst() + ": " + + StringUtils.stringifyException(e)); + } + } catch (RemoteException e) { + if (e.getMessage().indexOf(NotServingRegionException.class.getName()) >= 0) { + if (LOG.isDebugEnabled()) { + LOG.debug("Trying to get compaction state of " + pair.getFirst() + ": " + + StringUtils.stringifyException(e)); + } + } else { + throw e; + } + } + } + } catch (ServiceException se) { + throw ProtobufUtil.getRemoteException(se); + } finally { + zookeeper.close(); + } + break; } + return state; } /** @@ -4656,4 +4622,10 @@ public class HBaseAdmin implements Admin { throw e; } } + + private HRegionInfo getMobRegionInfo(TableName tableName) { + return new HRegionInfo(tableName, Bytes.toBytes(".mob"), + HConstants.EMPTY_END_ROW, false, 0); + } + } diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/CompactMobAction.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/CompactMobAction.java index 87c6dee..c93ba71 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/CompactMobAction.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/CompactMobAction.java @@ -51,9 +51,9 @@ public class CompactMobAction extends Action { LOG.info("Performing action: Compact mob of table " + tableName + ", major=" + major); try { if (major) { - admin.majorCompactMobs(tableName); + admin.majorCompact(tableName, Admin.CompactType.MOB); } else { - admin.compactMobs(tableName); + admin.compact(tableName, Admin.CompactType.MOB); } } catch (Exception ex) { LOG.warn("Mob Compaction failed, might be caused by other chaos: " + ex.getMessage()); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java index fc03c77..fbd81c9 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java @@ -570,7 +570,7 @@ public class TestMobCompactor { int largeFilesCount = countLargeFiles(5000, family1); // do the mob compaction - admin.compactMob(tableName, hcd1.getName()); + admin.compact(tableName, hcd1.getName(), Admin.CompactType.MOB); waitUntilMobCompactionFinished(tableName); assertEquals("After compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum), @@ -618,7 +618,7 @@ public class TestMobCompactor { countFiles(tableName, false, family2)); // do the major mob compaction, it will force all files to compaction - admin.majorCompactMob(tableName, hcd1.getName()); + admin.majorCompact(tableName, hcd1.getName(), Admin.CompactType.MOB); waitUntilMobCompactionFinished(tableName); assertEquals("After compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum), @@ -657,7 +657,7 @@ public class TestMobCompactor { Cell cell = result.getColumnLatestCell(hcd1.getName(), Bytes.toBytes(qf1)); assertEquals("Before compaction: mob value of k0", newValue0, Bytes.toString(CellUtil.cloneValue(cell))); - admin.majorCompactMob(tableName, hcd1.getName()); + admin.majorCompact(tableName, hcd1.getName(), Admin.CompactType.MOB); waitUntilMobCompactionFinished(tableName); // read the latest cell of key0, the cell seqId in bulk loaded file is not reset in the // scanner. The cell that has "new" value is still visible. @@ -705,7 +705,7 @@ public class TestMobCompactor { loadData(admin, bufMut, tableName, new Put[] { put1 }); // now two mob files admin.majorCompact(tableName); waitUntilCompactionFinished(tableName); - admin.majorCompactMob(tableName, hcd1.getName()); + admin.majorCompact(tableName, hcd1.getName(), Admin.CompactType.MOB); waitUntilMobCompactionFinished(tableName); // read the latest cell of key1. Get get = new Get(key1); @@ -731,12 +731,12 @@ public class TestMobCompactor { private void waitUntilMobCompactionFinished(TableName tableName) throws IOException, InterruptedException { long finished = EnvironmentEdgeManager.currentTime() + 60000; - CompactionState state = admin.getMobCompactionState(tableName); + CompactionState state = admin.getCompactionState(tableName, Admin.CompactType.MOB); while (EnvironmentEdgeManager.currentTime() < finished) { if (state == CompactionState.NONE) { break; } - state = admin.getMobCompactionState(tableName); + state = admin.getCompactionState(tableName, Admin.CompactType.MOB); Thread.sleep(10); } assertEquals(CompactionState.NONE, state); diff --git a/hbase-shell/src/main/ruby/hbase/admin.rb b/hbase-shell/src/main/ruby/hbase/admin.rb index 3afc807..8d7e73b 100644 --- a/hbase-shell/src/main/ruby/hbase/admin.rb +++ b/hbase-shell/src/main/ruby/hbase/admin.rb @@ -1013,10 +1013,10 @@ module Hbase # Requests a mob file compaction def compact_mob(table_name, family = nil) if family == nil - @admin.compactMobs(org.apache.hadoop.hbase.TableName.valueOf(table_name)) + @admin.compact(org.apache.hadoop.hbase.TableName.valueOf(table_name), org.apache.hadoop.hbase.client.Admin.CompactType.MOB) else # We are compacting a mob column family within a table. - @admin.compactMob(org.apache.hadoop.hbase.TableName.valueOf(table_name), family.to_java_bytes) + @admin.compact(org.apache.hadoop.hbase.TableName.valueOf(table_name), family.to_java_bytes, org.apache.hadoop.hbase.client.Admin.CompactType.MOB) end end @@ -1024,10 +1024,10 @@ module Hbase # Requests a mob file major compaction def major_compact_mob(table_name, family = nil) if family == nil - @admin.majorCompactMobs(org.apache.hadoop.hbase.TableName.valueOf(table_name)) + @admin.majorCompact(org.apache.hadoop.hbase.TableName.valueOf(table_name), org.apache.hadoop.hbase.client.Admin.CompactType.MOB) else # We are major compacting a mob column family within a table. - @admin.majorCompactMob(org.apache.hadoop.hbase.TableName.valueOf(table_name), family.to_java_bytes) + @admin.majorCompact(org.apache.hadoop.hbase.TableName.valueOf(table_name), family.to_java_bytes, org.apache.hadoop.hbase.client.Admin.CompactType.MOB) end end -- 1.9.3 (Apple Git-50)