From b9c22bb3e77b7fd685790d4d9f72973cf5483593 Mon Sep 17 00:00:00 2001 From: Zach York Date: Thu, 9 Mar 2017 00:17:41 -0800 Subject: [PATCH] HBASE-18840 Add functionality to sync meta table at master startup --- .../java/org/apache/hadoop/hbase/HRegionInfo.java | 2 +- .../org/apache/hadoop/hbase/MetaTableAccessor.java | 19 +++++++++ .../org/apache/hadoop/hbase/master/HMaster.java | 14 +++++++ .../java/org/apache/hadoop/hbase/util/FSUtils.java | 46 +++++++++++++++++++- .../org/apache/hadoop/hbase/util/TestFSUtils.java | 49 ++++++++++++++++++++++ 5 files changed, 127 insertions(+), 3 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java index d470ffa..78c09a8 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java @@ -714,7 +714,7 @@ public class HRegionInfo implements Comparable { /** @return true if this region is a meta region */ public boolean isMetaRegion() { - return tableName.equals(HRegionInfo.FIRST_META_REGIONINFO.getTable()); + return tableName.isMeta(); } /** diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java index bd8968d..255c155 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java @@ -23,6 +23,7 @@ import java.io.InterruptedIOException; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -1056,6 +1057,24 @@ public class MetaTableAccessor { } /** + * Scans the meta table and parses all HRegionInfos from the result. + * @param connection connection used to access the meta table + * @return A set of all HRegionInfos in the meta table + * @throws IOException + */ + public static Set getAllHRegionInfos(Connection connection) throws IOException { + Set regionInfos = new HashSet<>(); + List results = fullScan(connection, QueryType.ALL); + for (Result r : results) { + HRegionInfo info = getHRegionInfo(r); + if (info != null) { + regionInfos.add(info); + } + } + return regionInfos; + } + + /** * Returns the daughter regions by reading the corresponding columns of the catalog table * Result. * @param data a Result object from the catalog table scan diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index ad304ae..100cc44 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -282,6 +282,8 @@ public class HMaster extends HRegionServer implements MasterServices { } } + public static final String META_REFRESH_ON_STARTUP_KEY = "hbase.meta.startup.refresh"; + // MASTER is name of the webapp and the attribute name used stuffing this //instance into web context. public static final String MASTER = "master"; @@ -676,6 +678,10 @@ public class HMaster extends HRegionServer implements MasterServices { return MasterDumpServlet.class; } + public boolean getMetaRefreshOnStartup() { + return conf.getBoolean(META_REFRESH_ON_STARTUP_KEY, false); + } + @Override public MetricsMaster getMasterMetrics() { return metricsMaster; @@ -850,6 +856,14 @@ public class HMaster extends HRegionServer implements MasterServices { // assigned when master is shutting down if (isStopped()) return; + if (getMetaRefreshOnStartup()) { + try { + FSUtils.refreshMetaTableWithStorage(clusterConnection, fs, getRootDir()); + } catch (IOException e) { + LOG.warn("Failed to automatically add pre-existing tables to the meta table: ", e); + } + } + //Initialize after meta as it scans meta if (favoredNodesManager != null) { SnapshotOfRegionAssignmentFromMeta snapshotOfRegionAssignment = diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java index 6a6d689..1b46f39 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java @@ -23,6 +23,7 @@ import org.apache.hadoop.hbase.shaded.com.google.common.base.Throwables; import org.apache.hadoop.hbase.shaded.com.google.common.collect.Iterators; import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists; import org.apache.hadoop.hbase.shaded.com.google.common.primitives.Ints; +import com.google.common.collect.Sets; import edu.umd.cs.findbugs.annotations.CheckForNull; @@ -43,11 +44,13 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.Iterator; +import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Vector; +import java.util.Set; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; @@ -80,11 +83,14 @@ import org.apache.hadoop.hbase.HDFSBlocksDistribution; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.MetaTableAccessor; +import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.fs.HFileSystem; import org.apache.hadoop.hbase.io.HFileLink; import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.apache.hadoop.hbase.security.AccessDeniedException; import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; @@ -2367,7 +2373,7 @@ public abstract class FSUtils { // to the DFS FS instance and make the method getHedgedReadMetrics accessible, then invoke it // to get the singleton instance of DFSHedgedReadMetrics shared by DFSClients. final String name = "getHedgedReadMetrics"; - DFSClient dfsclient = ((DistributedFileSystem)FileSystem.get(c)).getClient(); + DFSClient dfsclient = ((DistributedFileSystem) FileSystem.get(c)).getClient(); Method m; try { m = dfsclient.getClass().getDeclaredMethod(name); @@ -2382,7 +2388,7 @@ public abstract class FSUtils { } m.setAccessible(true); try { - return (DFSHedgedReadMetrics)m.invoke(dfsclient); + return (DFSHedgedReadMetrics) m.invoke(dfsclient); } catch (IllegalAccessException e) { LOG.warn("Failed invoking method " + name + " on dfsclient; no hedged read metrics: " + e.getMessage()); @@ -2397,4 +2403,40 @@ public abstract class FSUtils { return null; } } + + /** + * Adds and removes regions from the meta table based on what is found on disk. + * @param clusterConnection connection used to access and update the meta table + * @param fs root directory filesystem. Used for determining Regions on disk. + * @param rootDir root directory to determine Regions on disk. + * @throws IOException + */ + public static void refreshMetaTableWithStorage(Connection clusterConnection, FileSystem fs, Path rootDir) + throws IOException { + Set existingRegionsInMeta = MetaTableAccessor.getAllHRegionInfos(clusterConnection); + Set existingRegionsOnFS = getAllRegionsOnFileSystem(fs, rootDir); + List regionsToRemove = new ArrayList<>(Sets.difference(existingRegionsInMeta, existingRegionsOnFS)); + List regionsToAdd = new ArrayList<>(Sets.difference(existingRegionsOnFS, existingRegionsInMeta)); + MetaTableAccessor.mutateRegions(clusterConnection, regionsToRemove, regionsToAdd); + } + + /** + * Determines all HRegionInfos under the root directory. + * @param fs the root directory FileSystem. + * @param rootDir the root directory used to determine Regions on disk. + * @return A set of all the HRegionInfos on disk. + * @throws IOException + */ + public static Set getAllRegionsOnFileSystem(FileSystem fs, Path rootDir) throws IOException { + Set hRegionInfos = new HashSet<>(); + for (Path tableDir: FSUtils.getTableDirs(fs, rootDir)) { + for (Path regionDir: FSUtils.getRegionDirs(fs, tableDir)) { + HRegionInfo info = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir); + if (info != null && !ReadReplicaClustersTableNameUtil.isMetaTableNameWithoutSuffix(info.getTable())) { + hRegionInfos.add(info); + } + } + } + return hRegionInfos; + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSUtils.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSUtils.java index 548c023..feec10c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSUtils.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSUtils.java @@ -40,8 +40,18 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HDFSBlocksDistribution; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.MetaTableAccessor; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.fs.HFileSystem; +import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hdfs.DFSConfigKeys; @@ -560,6 +570,45 @@ public class TestFSUtils { }*/ } + @Test + public void testRefreshMetaTableWithStorage() throws Exception { + HBaseTestingUtility htu = new HBaseTestingUtility(); + Configuration conf = htu.getConfiguration(); + htu.startMiniCluster(); + FileSystem fs = htu.getTestFileSystem(); + Connection clusterConnection = htu.getConnection(); + ColumnFamilyDescriptor cfd = ColumnFamilyDescriptorBuilder.newBuilder("c1".getBytes()).build(); + + Path rootDir = htu.getDataTestDirOnTestFS(); + HRegionInfo info1 = new HRegionInfo(TableName.valueOf("t1")); + TableDescriptor td = TableDescriptorBuilder.newBuilder(TableName.valueOf("t1")).addColumnFamily(cfd).build(); + HRegion region1 = htu.createRegionAndWAL(info1, rootDir, conf, td); + + HRegionInfo info2 = new HRegionInfo(TableName.valueOf("t2")); + td = TableDescriptorBuilder.newBuilder(TableName.valueOf("t2")).addColumnFamily(cfd).build(); + HRegion region2 = htu.createRegionAndWAL(info2, rootDir, conf, td); + + HRegionInfo info3 = new HRegionInfo(TableName.valueOf("t3")); + + // Modify Meta table to contain only info2 and info3 (createHRegion doesn't add tables to meta) + MetaTableAccessor.addRegionToMeta(clusterConnection, info2); + MetaTableAccessor.addRegionToMeta(clusterConnection, info3); + + assertTrue(htu.getMetaTableRows(info1.getTable()).isEmpty()); + assertFalse(htu.getMetaTableRows(info2.getTable()).isEmpty()); + assertFalse(htu.getMetaTableRows(info3.getTable()).isEmpty()); + + FSUtils.refreshMetaTableWithStorage(clusterConnection, fs, rootDir); + + assertFalse(htu.getMetaTableRows(info1.getTable()).isEmpty()); + assertFalse(htu.getMetaTableRows(info2.getTable()).isEmpty()); + assertTrue(htu.getMetaTableRows(info3.getTable()).isEmpty()); + + htu.closeRegionAndWAL(region1); + htu.closeRegionAndWAL(region2); + htu.shutdownMiniCluster(); + } + private void cleanupFile(FileSystem fileSys, Path name) throws IOException { assertTrue(fileSys.exists(name)); assertTrue(fileSys.delete(name, true)); -- 2.6.4