From 739d262b8475b3f053888c963c0ab5916d8b5981 Mon Sep 17 00:00:00 2001 From: Zach York Date: Thu, 9 Mar 2017 00:17:41 -0800 Subject: [PATCH] HBASE-18840 Add functionality to sync meta table at master startup --- .../java/org/apache/hadoop/hbase/HRegionInfo.java | 2 +- .../org/apache/hadoop/hbase/MetaTableAccessor.java | 53 +++++++++++++++++++++- .../org/apache/hadoop/hbase/master/HMaster.java | 15 ++++++ .../java/org/apache/hadoop/hbase/util/FSUtils.java | 53 ++++++++++++++++++++-- .../org/apache/hadoop/hbase/util/TestFSUtils.java | 49 ++++++++++++++++++++ 5 files changed, 167 insertions(+), 5 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java index fc03926..5186d7c 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java @@ -599,7 +599,7 @@ public class HRegionInfo implements RegionInfo, Comparable { */ @Override public boolean isMetaRegion() { - return tableName.equals(HRegionInfo.FIRST_META_REGIONINFO.getTable()); + return tableName.isMeta(); } /** diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java index 60afaca..10d8d11 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java @@ -26,6 +26,7 @@ import java.io.InterruptedIOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; @@ -1033,6 +1034,24 @@ public class MetaTableAccessor { } /** + * Scans the meta table and parses all HRegionInfos from the result. + * @param connection connection used to access the meta table + * @return A set of all HRegionInfos in the meta table + * @throws IOException if there is an error scanning meta + */ + public static Set getAllHRegionInfos(Connection connection) throws IOException { + Set regionInfos = new HashSet<>(); + List results = fullScan(connection, QueryType.ALL); + for (Result r : results) { + RegionInfo info = getRegionInfo(r); + if (info != null) { + regionInfos.add(info); + } + } + return regionInfos; + } + + /** * Returns the daughter regions by reading the corresponding columns of the catalog table * Result. * @param data a Result object from the catalog table scan @@ -1805,7 +1824,7 @@ public class MetaTableAccessor { * Deletes the specified region from META. * @param connection connection we're using * @param regionInfo region to be deleted from META - * @throws IOException + * @throws IOException if the delete operation fails on meta */ public static void deleteRegion(Connection connection, RegionInfo regionInfo) throws IOException { long time = EnvironmentEdgeManager.currentTime(); @@ -2138,6 +2157,38 @@ public class MetaTableAccessor { return list; } + /** + * Adds and Removes the specified regions from hbase:meta + * + * @param connection connection we're using + * @param regionsToRemove list of regions to be deleted from META + * @param regionsToAdd list of regions to be added to META + * @throws IOException + */ + public static void mutateRegions(Connection connection, + final List regionsToRemove, + final List regionsToAdd) + throws IOException { + List mutation = new ArrayList<>(); + if (regionsToRemove != null) { + for (RegionInfo hri : regionsToRemove) { + mutation.add(makeDeleteFromRegionInfo(hri, EnvironmentEdgeManager.currentTime())); + } + } + if (regionsToAdd != null) { + for (RegionInfo hri : regionsToAdd) { + mutation.add(makePutFromRegionInfo(hri, EnvironmentEdgeManager.currentTime())); + } + } + mutateMetaTable(connection, mutation); + if (regionsToRemove != null && regionsToRemove.size() > 0) { + LOG.debug("Deleted " + RegionInfo.getShortNameToLog(regionsToRemove)); + } + if (regionsToAdd != null && regionsToAdd.size() > 0) { + LOG.debug("Added " + RegionInfo.getShortNameToLog(regionsToAdd)); + } + } + private static void debugLogMutations(List mutations) throws IOException { if (!METALOG.isDebugEnabled()) { return; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index f1bec35..d977ba9 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -188,6 +188,7 @@ import org.apache.hadoop.hbase.util.Addressing; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.CompressionTest; import org.apache.hadoop.hbase.util.EncryptionTest; +import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.HFileArchiveUtil; import org.apache.hadoop.hbase.util.HasThread; import org.apache.hadoop.hbase.util.IdLock; @@ -295,6 +296,8 @@ public class HMaster extends HRegionServer implements MasterServices { } } + public static final String META_REFRESH_ON_STARTUP_KEY = "hbase.meta.startup.refresh"; + // MASTER is name of the webapp and the attribute name used stuffing this //instance into web context. public static final String MASTER = "master"; @@ -722,6 +725,10 @@ public class HMaster extends HRegionServer implements MasterServices { return MasterDumpServlet.class; } + public boolean getMetaRefreshOnStartup() { + return conf.getBoolean(META_REFRESH_ON_STARTUP_KEY, false); + } + @Override public MetricsMaster getMasterMetrics() { return metricsMaster; @@ -966,6 +973,14 @@ public class HMaster extends HRegionServer implements MasterServices { return; } + if (getMetaRefreshOnStartup()) { + try { + FSUtils.refreshMetaTableWithStorage(clusterConnection, fs, getRootDir()); + } catch (IOException e) { + LOG.warn("Failed to automatically add pre-existing tables to the meta table: ", e); + } + } + //Initialize after meta as it scans meta if (favoredNodesManager != null) { SnapshotOfRegionAssignmentFromMeta snapshotOfRegionAssignment = diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java index 5b968db..83b0bb4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java @@ -33,11 +33,13 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Set; import java.util.Vector; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.ConcurrentHashMap; @@ -67,11 +69,14 @@ import org.apache.hadoop.hbase.HDFSBlocksDistribution; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.fs.HFileSystem; import org.apache.hadoop.hbase.io.HFileLink; import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.MetaTableAccessor; import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.apache.hadoop.hbase.security.AccessDeniedException; import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter; @@ -92,6 +97,7 @@ import org.slf4j.LoggerFactory; import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hbase.thirdparty.com.google.common.base.Throwables; import org.apache.hbase.thirdparty.com.google.common.collect.Iterators; +import org.apache.hbase.thirdparty.com.google.common.collect.Sets; import org.apache.hbase.thirdparty.com.google.common.primitives.Ints; import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; @@ -1726,7 +1732,7 @@ public abstract class FSUtils extends CommonFSUtils { // to the DFS FS instance and make the method getHedgedReadMetrics accessible, then invoke it // to get the singleton instance of DFSHedgedReadMetrics shared by DFSClients. final String name = "getHedgedReadMetrics"; - DFSClient dfsclient = ((DistributedFileSystem)FileSystem.get(c)).getClient(); + DFSClient dfsclient = ((DistributedFileSystem) FileSystem.get(c)).getClient(); Method m; try { m = dfsclient.getClass().getDeclaredMethod(name); @@ -1741,7 +1747,7 @@ public abstract class FSUtils extends CommonFSUtils { } m.setAccessible(true); try { - return (DFSHedgedReadMetrics)m.invoke(dfsclient); + return (DFSHedgedReadMetrics) m.invoke(dfsclient); } catch (IllegalAccessException e) { LOG.warn("Failed invoking method " + name + " on dfsclient; no hedged read metrics: " + e.getMessage()); @@ -1787,7 +1793,7 @@ public abstract class FSUtils extends CommonFSUtils { FileStatus[] subPaths = srcFS.listStatus(src); for (FileStatus subPath : subPaths) { traversedPaths.addAll(copyFiles(srcFS, subPath.getPath(), dstFS, - new Path(dst, subPath.getPath().getName()), conf, pool, futures)); + new Path(dst, subPath.getPath().getName()), conf, pool, futures)); } } else { Future future = pool.submit(() -> { @@ -1798,4 +1804,45 @@ public abstract class FSUtils extends CommonFSUtils { } return traversedPaths; } + + /** + * Adds and removes regions from the meta table based on what is found on disk. + * @param clusterConnection connection used to access and update the meta table + * @param fs root directory filesystem. Used for determining Regions on disk. + * @param rootDir root directory to determine Regions on disk. + * @throws IOException if there is any issue interacting with the FS or meta table + */ + public static void refreshMetaTableWithStorage(Connection clusterConnection, + FileSystem fs, Path rootDir) + throws IOException { + Set existingRegionsInMeta = MetaTableAccessor.getAllHRegionInfos(clusterConnection); + Set existingRegionsOnFS = getAllRegionsOnFileSystem(fs, rootDir); + List regionsToRemove = new ArrayList<>(Sets.difference( + existingRegionsInMeta, existingRegionsOnFS)); + List regionsToAdd = new ArrayList<>(Sets.difference( + existingRegionsOnFS, existingRegionsInMeta)); + MetaTableAccessor.mutateRegions(clusterConnection, regionsToRemove, regionsToAdd); + } + + /** + * Determines all HRegionInfos under the root directory. + * @param fs the root directory FileSystem. + * @param rootDir the root directory used to determine Regions on disk. + * @return A set of all the HRegionInfos on disk. + * @throws IOException if there is a failure reading RegionInfo from FS + */ + public static Set getAllRegionsOnFileSystem(FileSystem fs, Path rootDir) + throws IOException { + Set hRegionInfos = new HashSet<>(); + for (Path tableDir: FSUtils.getTableDirs(fs, rootDir)) { + for (Path regionDir: FSUtils.getRegionDirs(fs, tableDir)) { + RegionInfo info = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir); + if (info != null && + !ReadReplicaClustersTableNameUtil.isMetaTableNameWithoutSuffix(info.getTable())) { + hRegionInfos.add(info); + } + } + } + return hRegionInfos; + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSUtils.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSUtils.java index c5863f7..e565f26 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSUtils.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSUtils.java @@ -39,8 +39,17 @@ import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HDFSBlocksDistribution; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.MetaTableAccessor; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.fs.HFileSystem; +import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hdfs.DFSConfigKeys; @@ -596,6 +605,46 @@ public class TestFSUtils { }*/ } + @Test + public void testRefreshMetaTableWithStorage() throws Exception { + HBaseTestingUtility htu = new HBaseTestingUtility(); + Configuration conf = htu.getConfiguration(); + htu.startMiniCluster(); + FileSystem fs = htu.getTestFileSystem(); + Connection clusterConnection = htu.getConnection(); + ColumnFamilyDescriptor cfd = ColumnFamilyDescriptorBuilder.newBuilder("c1".getBytes()).build(); + + Path rootDir = htu.getDataTestDirOnTestFS(); + HRegionInfo info1 = new HRegionInfo(TableName.valueOf("t1")); + TableDescriptor td = TableDescriptorBuilder.newBuilder( + TableName.valueOf("t1")).setColumnFamily(cfd).build(); + HRegion region1 = htu.createRegionAndWAL(info1, rootDir, conf, td); + + HRegionInfo info2 = new HRegionInfo(TableName.valueOf("t2")); + td = TableDescriptorBuilder.newBuilder(TableName.valueOf("t2")).setColumnFamily(cfd).build(); + HRegion region2 = htu.createRegionAndWAL(info2, rootDir, conf, td); + + HRegionInfo info3 = new HRegionInfo(TableName.valueOf("t3")); + + // Modify Meta table to contain only info2 and info3 (createHRegion doesn't add tables to meta) + MetaTableAccessor.addRegionToMeta(clusterConnection, info2); + MetaTableAccessor.addRegionToMeta(clusterConnection, info3); + + assertTrue(htu.getMetaTableRows(info1.getTable()).isEmpty()); + assertFalse(htu.getMetaTableRows(info2.getTable()).isEmpty()); + assertFalse(htu.getMetaTableRows(info3.getTable()).isEmpty()); + + FSUtils.refreshMetaTableWithStorage(clusterConnection, fs, rootDir); + + assertFalse(htu.getMetaTableRows(info1.getTable()).isEmpty()); + assertFalse(htu.getMetaTableRows(info2.getTable()).isEmpty()); + assertTrue(htu.getMetaTableRows(info3.getTable()).isEmpty()); + + htu.closeRegionAndWAL(region1); + htu.closeRegionAndWAL(region2); + htu.shutdownMiniCluster(); + } + private void cleanupFile(FileSystem fileSys, Path name) throws IOException { assertTrue(fileSys.exists(name)); assertTrue(fileSys.delete(name, true)); -- 2.6.4