Index: hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSTableDescriptors.java =================================================================== --- hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSTableDescriptors.java (revision 1454498) +++ hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSTableDescriptors.java (working copy) @@ -17,7 +17,12 @@ */ package org.apache.hadoop.hbase.util; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import java.io.FileNotFoundException; import java.io.IOException; @@ -28,7 +33,12 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.TableDescriptors; import org.apache.hadoop.hbase.exceptions.TableExistsException; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -266,5 +276,23 @@ } } + @Test + public void testCreateTableDescriptorUpdatesIfExistsAlready() throws IOException { + Path testdir = UTIL.getDataTestDir("testCreateTableDescriptorUpdatesIfThereExistsAlready"); + HTableDescriptor htd = new HTableDescriptor( + "testCreateTableDescriptorUpdatesIfThereExistsAlready"); + FileSystem fs = FileSystem.get(UTIL.getConfiguration()); + assertTrue(FSTableDescriptors.createTableDescriptor(fs, testdir, htd)); + assertFalse(FSTableDescriptors.createTableDescriptor(fs, testdir, htd)); + htd.setValue(Bytes.toBytes("mykey"), Bytes.toBytes("myValue")); + assertTrue(FSTableDescriptors.createTableDescriptor(fs, testdir, htd)); //this will re-create + Path tableDir = FSUtils.getTablePath(testdir, htd.getName()); + Path tmpTableDir = new Path(tableDir, ".tmp"); + FileStatus[] statuses = fs.listStatus(tmpTableDir); + assertTrue(statuses.length == 0); + + assertEquals(htd, FSTableDescriptors.getTableDescriptor(fs, tableDir)); + } + } Index: hbase-server/src/test/java/org/apache/hadoop/hbase/util/MockRegionServerServices.java =================================================================== --- hbase-server/src/test/java/org/apache/hadoop/hbase/util/MockRegionServerServices.java (revision 1454498) +++ hbase-server/src/test/java/org/apache/hadoop/hbase/util/MockRegionServerServices.java (working copy) @@ -47,7 +47,7 @@ public class MockRegionServerServices implements RegionServerServices { private final Map regions = new HashMap(); private boolean stopping = false; - private final ConcurrentSkipListMap rit = + private final ConcurrentSkipListMap rit = new ConcurrentSkipListMap(Bytes.BYTES_COMPARATOR); private HFileSystem hfs = null; private ZooKeeperWatcher zkw = null; @@ -86,7 +86,7 @@ } @Override - public void postOpenDeployTasks(HRegion r, CatalogTracker ct, boolean daughter) + public void postOpenDeployTasks(HRegion r, CatalogTracker ct) throws KeeperException, IOException { addToOnlineRegions(r); } @@ -125,7 +125,7 @@ public ZooKeeperWatcher getZooKeeper() { return zkw; } - + public RegionServerAccounting getRegionServerAccounting() { return null; } Index: hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java =================================================================== --- hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java (revision 1454498) +++ hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java (working copy) @@ -21,8 +21,8 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -34,25 +34,25 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.LargeTests; -import org.apache.hadoop.hbase.exceptions.MasterNotRunningException; import org.apache.hadoop.hbase.MiniHBaseCluster; import org.apache.hadoop.hbase.RegionTransition; import org.apache.hadoop.hbase.Server; import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.exceptions.UnknownRegionException; -import org.apache.hadoop.hbase.exceptions.ZooKeeperConnectionException; import org.apache.hadoop.hbase.catalog.MetaReader; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.exceptions.DeserializationException; +import org.apache.hadoop.hbase.exceptions.MasterNotRunningException; +import org.apache.hadoop.hbase.exceptions.UnknownRegionException; +import org.apache.hadoop.hbase.exceptions.ZooKeeperConnectionException; import org.apache.hadoop.hbase.executor.EventType; import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.master.RegionState; @@ -339,59 +339,6 @@ } /** - * Messy test that simulates case where SplitTransactions fails to add one - * of the daughters up into the .META. table before crash. We're testing - * fact that the shutdown handler will fixup the missing daughter region - * adding it back into .META. - * @throws IOException - * @throws InterruptedException - */ - @Test (timeout = 300000) public void testShutdownSimpleFixup() - throws IOException, InterruptedException, ServiceException { - final byte [] tableName = Bytes.toBytes("testShutdownSimpleFixup"); - - // Create table then get the single region for our new table. - HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY); - List regions = cluster.getRegions(tableName); - HRegionInfo hri = getAndCheckSingleTableRegion(regions); - - int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri); - - // Turn off balancer so it doesn't cut in and mess up our placements. - this.admin.setBalancerRunning(false, true); - // Turn off the meta scanner so it don't remove parent on us. - cluster.getMaster().setCatalogJanitorEnabled(false); - try { - // Add a bit of load up into the table so splittable. - TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY); - // Get region pre-split. - HRegionServer server = cluster.getRegionServer(tableRegionIndex); - printOutRegions(server, "Initial regions: "); - int regionCount = ProtobufUtil.getOnlineRegions(server).size(); - // Now split. - split(hri, server, regionCount); - // Get daughters - List daughters = checkAndGetDaughters(tableName); - // Remove one of the daughters from .META. to simulate failed insert of - // daughter region up into .META. - removeDaughterFromMeta(daughters.get(0).getRegionName()); - // Now crash the server - cluster.abortRegionServer(tableRegionIndex); - waitUntilRegionServerDead(); - awaitDaughters(tableName, daughters.size()); - // Assert daughters are online. - regions = cluster.getRegions(tableName); - for (HRegion r: regions) { - assertTrue(daughters.contains(r)); - } - } finally { - admin.setBalancerRunning(true, false); - cluster.getMaster().setCatalogJanitorEnabled(true); - t.close(); - } - } - - /** * Test that if daughter split on us, we won't do the shutdown handler fixup * just because we can't find the immediate daughter of an offlined parent. * @throws IOException Index: hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java =================================================================== --- hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java (revision 1454498) +++ hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java (working copy) @@ -118,7 +118,7 @@ // 3. finish phase II // note that this replicates some code from SplitTransaction // 2nd daughter first - server.postOpenDeployTasks(regions.getSecond(), server.getCatalogTracker(), true); + server.postOpenDeployTasks(regions.getSecond(), server.getCatalogTracker()); // Add to online regions server.addToOnlineRegions(regions.getSecond()); // THIS is the crucial point: @@ -128,7 +128,7 @@ assertTrue(test(con, tableName, lastRow, server)); // first daughter second - server.postOpenDeployTasks(regions.getFirst(), server.getCatalogTracker(), true); + server.postOpenDeployTasks(regions.getFirst(), server.getCatalogTracker()); // Add to online regions server.addToOnlineRegions(regions.getFirst()); assertTrue(test(con, tableName, firstRow, server)); Index: hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java =================================================================== --- hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java (revision 1454498) +++ hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java (working copy) @@ -291,7 +291,7 @@ } @Override - public void postOpenDeployTasks(HRegion r, CatalogTracker ct, boolean daughter) + public void postOpenDeployTasks(HRegion r, CatalogTracker ct) throws KeeperException, IOException { // TODO Auto-generated method stub } Index: hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterCoprocessorExceptionWithRemove.java =================================================================== --- hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterCoprocessorExceptionWithRemove.java (revision 1454498) +++ hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterCoprocessorExceptionWithRemove.java (working copy) @@ -19,11 +19,21 @@ package org.apache.hadoop.hbase.coprocessor; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + import java.io.IOException; -import java.io.InterruptedIOException; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.Abortable; +import org.apache.hadoop.hbase.CoprocessorEnvironment; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.MiniHBaseCluster; import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.master.MasterCoprocessorHost; @@ -35,8 +45,6 @@ import org.junit.Test; import org.junit.experimental.categories.Category; -import static org.junit.Assert.*; - /** * Tests unhandled exceptions thrown by coprocessors running on master. * Expected result is that the master will remove the buggy coprocessor from @@ -173,7 +181,7 @@ // In this test, there is only a single coprocessor (BuggyMasterObserver). String coprocessorName = BuggyMasterObserver.class.getName(); - assertTrue(master.getLoadedCoprocessors().equals("[" + coprocessorName + "]")); + assertTrue(master.getLoadedCoprocessors().contains(coprocessorName)); HTableDescriptor htd1 = new HTableDescriptor(TEST_TABLE1); htd1.addFamily(new HColumnDescriptor(TEST_FAMILY1)); @@ -201,7 +209,7 @@ masterTracker.masterZKNodeWasDeleted); String loadedCoprocessors = master.getLoadedCoprocessors(); - assertTrue(loadedCoprocessors.equals("[" + coprocessorName + "]")); + assertTrue(loadedCoprocessors.contains(coprocessorName)); Index: hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterCoprocessorExceptionWithAbort.java =================================================================== --- hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterCoprocessorExceptionWithAbort.java (revision 1454498) +++ hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterCoprocessorExceptionWithAbort.java (working copy) @@ -195,9 +195,7 @@ // (namely the part that shows the set of loaded coprocessors). // In this test, there is only a single coprocessor (BuggyMasterObserver). assertTrue(master.getLoadedCoprocessors(). - equals("[" + - TestMasterCoprocessorExceptionWithAbort.BuggyMasterObserver.class.getName() + - "]")); + contains(TestMasterCoprocessorExceptionWithAbort.BuggyMasterObserver.class.getName())); CreateTableThread createTableThread = new CreateTableThread(UTIL); Index: hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java (revision 1454498) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java (working copy) @@ -223,7 +223,7 @@ if (this.parent.getCoprocessorHost() != null) { this.parent.getCoprocessorHost().preSplit(); } - + // Coprocessor callback if (this.parent.getCoprocessorHost() != null) { this.parent.getCoprocessorHost().preSplit(this.splitrow); @@ -289,7 +289,7 @@ throw new IOException(exceptionToThrow); } - + if (hstoreFilesToSplit.size() == 0) { String errorMsg = "No store files to split for the region "+this.parent.getRegionInfo(); LOG.error(errorMsg); @@ -336,10 +336,14 @@ // HBase-4562). this.journal.add(JournalEntry.PONR); - // Edit parent in meta. Offlines parent region and adds splita and splitb. + // Edit parent in meta. Offlines parent region and adds splita and splitb + // as an atomic update. See HBASE-7721. This update to META makes the region + // will determine whether the region is split or not in case of failures. + // If it is successful, master will roll-forward, if not, master will rollback + // and assign the parent region. if (!testing) { - MetaEditor.offlineParentInMeta(server.getCatalogTracker(), - this.parent.getRegionInfo(), a.getRegionInfo(), b.getRegionInfo()); + MetaEditor.splitRegion(server.getCatalogTracker(), parent.getRegionInfo(), + a.getRegionInfo(), b.getRegionInfo(), server.getServerName()); } return new PairOfSameType(a, b); } @@ -389,10 +393,10 @@ if (services != null) { try { // add 2nd daughter first (see HBASE-4335) - services.postOpenDeployTasks(b, server.getCatalogTracker(), true); + services.postOpenDeployTasks(b, server.getCatalogTracker()); // Should add it to OnlineRegions services.addToOnlineRegions(b); - services.postOpenDeployTasks(a, server.getCatalogTracker(), true); + services.postOpenDeployTasks(a, server.getCatalogTracker()); services.addToOnlineRegions(a); } catch (KeeperException ke) { throw new IOException(ke); @@ -736,7 +740,7 @@ if (this.parent.getCoprocessorHost() != null) { this.parent.getCoprocessorHost().preRollBackSplit(); } - + boolean result = true; FileSystem fs = this.parent.getFilesystem(); ListIterator iterator = Index: hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java (revision 1454498) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java (working copy) @@ -317,7 +317,7 @@ public void run() { try { this.services.postOpenDeployTasks(this.region, - this.server.getCatalogTracker(), false); + this.server.getCatalogTracker()); } catch (Exception e) { LOG.warn("Exception running postOpenDeployTasks; region=" + this.region.getRegionInfo().getEncodedName(), e); Index: hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java (revision 1454498) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java (working copy) @@ -40,7 +40,7 @@ */ public boolean isStopping(); - /** @return the HLog for a particular region. Pass null for getting the + /** @return the HLog for a particular region. Pass null for getting the * default (common) WAL */ public HLog getWAL(HRegionInfo regionInfo) throws IOException; @@ -62,15 +62,13 @@ /** * Tasks to perform after region open to complete deploy of region on * regionserver - * + * * @param r Region to open. * @param ct Instance of {@link CatalogTracker} - * @param daughter True if this is daughter of a split * @throws KeeperException * @throws IOException */ - public void postOpenDeployTasks(final HRegion r, final CatalogTracker ct, - final boolean daughter) + public void postOpenDeployTasks(final HRegion r, final CatalogTracker ct) throws KeeperException, IOException; /** Index: hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (revision 1454498) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (working copy) @@ -1692,12 +1692,10 @@ } @Override - public void postOpenDeployTasks(final HRegion r, final CatalogTracker ct, - final boolean daughter) + public void postOpenDeployTasks(final HRegion r, final CatalogTracker ct) throws KeeperException, IOException { checkOpen(); - LOG.info("Post open deploy tasks for region=" + r.getRegionNameAsString() + - ", daughter=" + daughter); + LOG.info("Post open deploy tasks for region=" + r.getRegionNameAsString()); // Do checks to see if we need to compact (references or too many files) for (Store s : r.getStores().values()) { if (s.hasReferences() || s.needsCompaction()) { @@ -1718,17 +1716,11 @@ MetaEditor.updateMetaLocation(ct, r.getRegionInfo(), this.serverNameFromMasterPOV, openSeqNum); } else { - if (daughter) { - // If daughter of a split, update whole row, not just location. - MetaEditor.addDaughter(ct, r.getRegionInfo(), - this.serverNameFromMasterPOV, openSeqNum); - } else { - MetaEditor.updateRegionLocation(ct, r.getRegionInfo(), - this.serverNameFromMasterPOV, openSeqNum); - } + MetaEditor.updateRegionLocation(ct, r.getRegionInfo(), + this.serverNameFromMasterPOV, openSeqNum); } LOG.info("Done with post open deploy task for region=" + - r.getRegionNameAsString() + ", daughter=" + daughter); + r.getRegionNameAsString()); } Index: hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java (revision 1454498) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java (working copy) @@ -445,7 +445,8 @@ bootstrap(rd, c); } - // Create tableinfo-s for ROOT and META if not already there. + // Create tableinfo-s for ROOT and META if not already there. This also updates the + //descriptors if they are older versions. FSTableDescriptors.createTableDescriptor(fs, rd, HTableDescriptor.ROOT_TABLEDESC, false); FSTableDescriptors.createTableDescriptor(fs, rd, HTableDescriptor.META_TABLEDESC, false); Index: hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java (revision 1454498) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java (working copy) @@ -27,7 +27,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -753,10 +752,6 @@ this.balancer.setClusterStatus(getClusterStatus()); - // Fixing up missing daughters if any - status.setStatus("Fixing up missing daughters"); - fixupDaughters(status); - if (!masterRecovery) { // Start balancer and meta catalog janitor after meta and regions have // been assigned. @@ -944,41 +939,6 @@ } } - void fixupDaughters(final MonitoredTask status) throws IOException { - final Map offlineSplitParents = - new HashMap(); - // This visitor collects offline split parents in the .META. table - MetaReader.Visitor visitor = new MetaReader.Visitor() { - @Override - public boolean visit(Result r) throws IOException { - if (r == null || r.isEmpty()) return true; - HRegionInfo info = - HRegionInfo.getHRegionInfo(r); - if (info == null) return true; // Keep scanning - if (info.isOffline() && info.isSplit()) { - offlineSplitParents.put(info, r); - } - // Returning true means "keep scanning" - return true; - } - }; - // Run full scan of .META. catalog table passing in our custom visitor - MetaReader.fullScan(this.catalogTracker, visitor); - // Now work on our list of found parents. See if any we can clean up. - int fixups = 0; - for (Map.Entry e : offlineSplitParents.entrySet()) { - ServerName sn = HRegionInfo.getServerName(e.getValue()); - if (!serverManager.isServerDead(sn)) { // Otherwise, let SSH take care of it - fixups += ServerShutdownHandler.fixupDaughters( - e.getValue(), assignmentManager, catalogTracker); - } - } - if (fixups != 0) { - LOG.info("Scanned the catalog and fixed up " + fixups + - " missing daughter region(s)"); - } - } - /** * Split a server's log and expire it if we find it is one of the online * servers. Index: hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java (revision 1454498) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java (working copy) @@ -27,12 +27,10 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.Server; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.catalog.CatalogTracker; -import org.apache.hadoop.hbase.catalog.MetaEditor; import org.apache.hadoop.hbase.catalog.MetaReader; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.executor.EventHandler; @@ -43,8 +41,6 @@ import org.apache.hadoop.hbase.master.RegionState; import org.apache.hadoop.hbase.master.RegionStates; import org.apache.hadoop.hbase.master.ServerManager; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.PairOfSameType; import org.apache.hadoop.hbase.zookeeper.ZKAssign; import org.apache.zookeeper.KeeperException; @@ -310,12 +306,9 @@ return false; } if (hri.isOffline() && hri.isSplit()) { - LOG.debug("Offlined and split region " + hri.getRegionNameAsString() + - "; checking daughter presence"); - if (MetaReader.getRegion(catalogTracker, hri.getRegionName()) == null) { - return false; - } - fixupDaughters(result, assignmentManager, catalogTracker); + //HBASE-7721: Split parent and daughters are inserted into META as an atomic operation. + //If the meta scanner saw the parent split, then it should see the daughters as assigned + //to the dead server. We don't have to do anything. return false; } boolean disabling = assignmentManager.getZKTable().isDisablingTable( @@ -327,124 +320,4 @@ } return true; } - - /** - * Check that daughter regions are up in .META. and if not, add them. - * @param result The contents of the parent row in .META. - * @param assignmentManager - * @param catalogTracker - * @return the number of daughters missing and fixed - * @throws IOException - */ - public static int fixupDaughters(final Result result, - final AssignmentManager assignmentManager, - final CatalogTracker catalogTracker) - throws IOException { - PairOfSameType daughters = HRegionInfo.getDaughterRegions(result); - int fixedA = fixupDaughter(result, daughters.getFirst(), - assignmentManager, catalogTracker); - int fixedB = fixupDaughter(result, daughters.getSecond(), - assignmentManager, catalogTracker); - return fixedA + fixedB; - } - - /** - * Check individual daughter is up in .META.; fixup if its not. - * @param result The contents of the parent row in .META. - not used - * @param daughter Which daughter to check for. - * @return 1 if the daughter is missing and fixed. Otherwise 0 - * @throws IOException - */ - static int fixupDaughter(final Result result, HRegionInfo daughter, - final AssignmentManager assignmentManager, - final CatalogTracker catalogTracker) - throws IOException { - if (daughter == null) return 0; - if (isDaughterMissing(catalogTracker, daughter)) { - LOG.info("Fixup; missing daughter " + daughter.getRegionNameAsString()); - MetaEditor.addDaughter(catalogTracker, daughter, null, HConstants.NO_SEQNUM); - - // TODO: Log WARN if the regiondir does not exist in the fs. If its not - // there then something wonky about the split -- things will keep going - // but could be missing references to parent region. - - // And assign it. - assignmentManager.assign(daughter, true, true); - return 1; - } else { - LOG.debug("Daughter " + daughter.getRegionNameAsString() + " present"); - } - return 0; - } - - /** - * Look for presence of the daughter OR of a split of the daughter in .META. - * Daughter could have been split over on regionserver before a run of the - * catalogJanitor had chance to clear reference from parent. - * @param daughter Daughter region to search for. - * @throws IOException - */ - private static boolean isDaughterMissing(final CatalogTracker catalogTracker, - final HRegionInfo daughter) throws IOException { - FindDaughterVisitor visitor = new FindDaughterVisitor(daughter); - // Start the scan at what should be the daughter's row in the .META. - // We will either 1., find the daughter or some derivative split of the - // daughter (will have same table name and start row at least but will sort - // after because has larger regionid -- the regionid is timestamp of region - // creation), OR, we will not find anything with same table name and start - // row. If the latter, then assume daughter missing and do fixup. - byte [] startrow = daughter.getRegionName(); - MetaReader.fullScan(catalogTracker, visitor, startrow); - return !visitor.foundDaughter(); - } - - /** - * Looks for daughter. Sets a flag if daughter or some progeny of daughter - * is found up in .META.. - */ - static class FindDaughterVisitor implements MetaReader.Visitor { - private final HRegionInfo daughter; - private boolean found = false; - - FindDaughterVisitor(final HRegionInfo daughter) { - this.daughter = daughter; - } - - /** - * @return True if we found a daughter region during our visiting. - */ - boolean foundDaughter() { - return this.found; - } - - @Override - public boolean visit(Result r) throws IOException { - HRegionInfo hri = - HRegionInfo.getHRegionInfo(r); - if (hri == null) { - LOG.warn("No serialized HRegionInfo in " + r); - return true; - } - byte [] value = r.getValue(HConstants.CATALOG_FAMILY, - HConstants.SERVER_QUALIFIER); - // See if daughter is assigned to some server - if (value == null) return false; - - // Now see if we have gone beyond the daughter's startrow. - if (!Bytes.equals(daughter.getTableName(), - hri.getTableName())) { - // We fell into another table. Stop scanning. - return false; - } - // If our start rows do not compare, move on. - if (!Bytes.equals(daughter.getStartKey(), hri.getStartKey())) { - return false; - } - // Else, table name and start rows compare. It means that the daughter - // or some derivative split of the daughter is up in .META. Daughter - // exists. - this.found = true; - return false; - } - } } Index: hbase-server/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java (revision 1454498) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java (working copy) @@ -34,8 +34,15 @@ import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel; +import org.apache.hadoop.hbase.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.Mutate.MutateType; +import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutation.MultiMutateRequest; +import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutation.MultiRowMutationService; import org.apache.hadoop.hbase.util.Bytes; +import com.google.protobuf.ServiceException; + /** * Writes region and assignment information to .META.. * TODO: Put MetaReader and MetaEditor together; doesn't make sense having @@ -210,8 +217,8 @@ * Adds a (single) META row for the specified new region and its daughters. Note that this does * not add its daughter's as different rows, but adds information about the daughters * in the same row as the parent. Use - * {@link #offlineParentInMeta(CatalogTracker, HRegionInfo, HRegionInfo, HRegionInfo)} and - * {@link #addDaughter(CatalogTracker, HRegionInfo, ServerName, long)} if you want to do that. + * {@link #splitRegion(CatalogTracker, HRegionInfo, HRegionInfo, HRegionInfo, ServerName)} + * if you want to do that. * @param meta the HTable for META * @param regionInfo region information * @param splitA first split daughter of the parent regionInfo @@ -246,32 +253,6 @@ } /** - * Offline parent in meta. - * Used when splitting. - * @param catalogTracker - * @param parent - * @param a Split daughter region A - * @param b Split daughter region B - * @throws NotAllMetaRegionsOnlineException - * @throws IOException - */ - public static void offlineParentInMeta(CatalogTracker catalogTracker, - HRegionInfo parent, final HRegionInfo a, final HRegionInfo b) - throws NotAllMetaRegionsOnlineException, IOException { - HRegionInfo copyOfParent = new HRegionInfo(parent); - copyOfParent.setOffline(true); - copyOfParent.setSplit(true); - HTable meta = MetaReader.getMetaHTable(catalogTracker); - try { - addRegionToMeta(meta, copyOfParent, a, b); - LOG.info("Offlined parent region " + parent.getRegionNameAsString() + - " in META"); - } finally { - meta.close(); - } - } - - /** * Adds a daughter region entry to meta. * @param regionInfo the region to put * @param sn the location of the region @@ -291,6 +272,60 @@ } /** + * Splits the region into two in an atomic operation. Offlines the parent + * region with the information that it is split into two, and also adds + * the daughter regions. Does not add the location information to the daughter + * regions since they are not open yet. + * @param catalogTracker the catalog tracker + * @param parent the parent region which is split + * @param splitA Split daughter region A + * @param splitB Split daughter region A + * @param sn the location of the region + */ + public static void splitRegion(final CatalogTracker catalogTracker, + HRegionInfo parent, HRegionInfo splitA, HRegionInfo splitB, + ServerName sn) throws IOException { + HTable meta = MetaReader.getMetaHTable(catalogTracker); + HRegionInfo copyOfParent = new HRegionInfo(parent); + copyOfParent.setOffline(true); + copyOfParent.setSplit(true); + + //Put for parent + Put putParent = makePutFromRegionInfo(copyOfParent); + addDaughtersToPut(putParent, splitA, splitB); + + //Puts for daughters + Put putA = makePutFromRegionInfo(splitA); + Put putB = makePutFromRegionInfo(splitB); + + addLocation(putA, sn, 1); //these are new regions, openSeqNum = 1 is fine. + addLocation(putB, sn, 1); + + byte[] tableRow = Bytes.toBytes(parent.getRegionNameAsString() + HConstants.DELIMITER); + multiPut(meta, tableRow, putParent, putA, putB); + } + + /** + * Performs an atomic multi-Put operation against the given table. + */ + private static void multiPut(HTable table, byte[] row, Put... puts) throws IOException { + CoprocessorRpcChannel channel = table.coprocessorService(row); + MultiMutateRequest.Builder mmrBuilder = MultiMutateRequest.newBuilder(); + for (Put put : puts) { + mmrBuilder.addMutationRequest(ProtobufUtil.toMutate(MutateType.PUT, put)); + } + + MultiRowMutationService.BlockingInterface service = + MultiRowMutationService.newBlockingStub(channel); + try { + service.mutateRows(null, mmrBuilder.build()); + } catch (ServiceException ex) { + ProtobufUtil.toIOException(ex); + } + } + + + /** * Updates the location of the specified META region in ROOT to be the * specified server hostname and startcode. *

Index: hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java (revision 1454498) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java (working copy) @@ -54,7 +54,7 @@ * passed filesystem. It expects descriptors to be in a file under the * table's directory in FS. Can be read-only -- i.e. does not modify * the filesystem or can be read and write. - * + * *

Also has utility for keeping up the table descriptors tableinfo file. * The table schema file is kept under the table directory in the filesystem. * It has a {@link #TABLEINFO_NAME} prefix and then a suffix that is the @@ -163,7 +163,7 @@ return cachedtdm.getTableDescriptor(); } } - + TableDescriptorModtime tdmt = null; try { tdmt = getTableDescriptorModtime(this.fs, this.rootdir, tablename); @@ -174,7 +174,7 @@ LOG.debug("Exception during readTableDecriptor. Current table name = " + tablename, ioe); } - + if (tdmt == null) { LOG.warn("The following folder is in HBase's root directory and " + "doesn't contain a table descriptor, " + @@ -241,7 +241,7 @@ /** * Checks if .tableinfo exists for given table - * + * * @param fs file system * @param rootdir root directory of HBase installation * @param tableName name of table @@ -488,7 +488,7 @@ * @param tableDir * @param status * @return Descriptor file or null if we failed write. - * @throws IOException + * @throws IOException */ private static Path writeTableDescriptor(final FileSystem fs, final HTableDescriptor hTableDescriptor, final Path tableDir, @@ -554,7 +554,7 @@ /** * Create new HTableDescriptor in HDFS. Happens when we are creating table. - * + * * @param htableDescriptor * @param conf */ @@ -568,7 +568,7 @@ * Create new HTableDescriptor in HDFS. Happens when we are creating table. If * forceCreation is true then even if previous table descriptor is present it * will be overwritten - * + * * @param htableDescriptor * @param conf * @param forceCreation True if we are to overwrite existing file. @@ -597,7 +597,7 @@ * Create new HTableDescriptor in HDFS. Happens when we are creating table. If * forceCreation is true then even if previous table descriptor is present it * will be overwritten - * + * * @param fs * @param htableDescriptor * @param rootdir @@ -630,8 +630,10 @@ LOG.info("Current tableInfoPath = " + status.getPath()); if (!forceCreation) { if (fs.exists(status.getPath()) && status.getLen() > 0) { - LOG.info("TableInfo already exists.. Skipping creation"); - return false; + if (getTableDescriptor(fs, status.getPath().getParent()).equals(htableDescriptor)) { + LOG.info("TableInfo already exists.. Skipping creation"); + return false; + } } } } Index: hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java (revision 1454498) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java (working copy) @@ -143,7 +143,7 @@ public static FSUtils getInstance(FileSystem fs, Configuration conf) { String scheme = fs.getUri().getScheme(); if (scheme == null) { - LOG.warn("Could not find scheme for uri " + + LOG.warn("Could not find scheme for uri " + fs.getUri() + ", default to hdfs"); scheme = "hdfs"; } @@ -189,7 +189,7 @@ *

  • use the default block size
  • *
  • not track progress
  • * - * + * * @param fs {@link FileSystem} on which to write the file * @param path {@link Path} to the file to write * @return output stream to the created file @@ -209,7 +209,7 @@ *
  • use the default block size
  • *
  • not track progress
  • * - * + * * @param fs {@link FileSystem} on which to write the file * @param path {@link Path} to the file to write * @param perm @@ -229,7 +229,7 @@ /** * Get the file permissions specified in the configuration, if they are * enabled. - * + * * @param fs filesystem that the file will be created on. * @param conf configuration to read for determining if permissions are * enabled and which to use @@ -321,11 +321,11 @@ } /** - * Check whether dfs is in safemode. + * Check whether dfs is in safemode. * @param conf * @throws IOException */ - public static void checkDfsSafeMode(final Configuration conf) + public static void checkDfsSafeMode(final Configuration conf) throws IOException { boolean isInSafeMode = false; FileSystem fs = FileSystem.get(conf); @@ -337,7 +337,7 @@ throw new IOException("File system is in safemode, it can't be written now"); } } - + /** * Verifies current version of file system * @@ -428,7 +428,7 @@ * @param message if true, issues a message on System.out * * @throws IOException e - * @throws DeserializationException + * @throws DeserializationException */ public static void checkVersion(FileSystem fs, Path rootdir, boolean message) throws IOException, DeserializationException { @@ -445,7 +445,7 @@ * @param retries number of times to retry * * @throws IOException e - * @throws DeserializationException + * @throws DeserializationException */ public static void checkVersion(FileSystem fs, Path rootdir, boolean message, int wait, int retries) @@ -610,7 +610,7 @@ /** * @param cid - * @throws IOException + * @throws IOException */ private static void rewriteAsPb(final FileSystem fs, final Path rootdir, final Path p, final ClusterId cid) @@ -780,9 +780,9 @@ * @param fs file system * @param status file status of the file * @param start start position of the portion - * @param length length of the portion + * @param length length of the portion * @return The HDFS blocks distribution - */ + */ static public HDFSBlocksDistribution computeHDFSBlocksDistribution( final FileSystem fs, FileStatus status, long start, long length) throws IOException { @@ -794,12 +794,12 @@ long len = bl.getLength(); blocksDistribution.addHostsAndBlockWeight(hosts, len); } - + return blocksDistribution; } - - + + /** * Runs through the hbase rootdir and checks all stores have only * one file in them -- that is, they've been major compacted. Looks @@ -868,7 +868,7 @@ * * @param master The master defining the HBase root and file system. * @return A map for each table and its percentage. - * + * * @throws IOException When scanning the directory fails. */ public static Map getTableFragmentation( @@ -1112,7 +1112,7 @@ } /** - * Recover file lease. Used when a file might be suspect + * Recover file lease. Used when a file might be suspect * to be had been left open by another process. * @param fs FileSystem handle * @param p Path of file to recover lease @@ -1121,7 +1121,7 @@ */ public abstract void recoverFileLease(final FileSystem fs, final Path p, Configuration conf) throws IOException; - + /** * @param fs * @param rootdir @@ -1288,10 +1288,10 @@ throws IOException { return getRootDir(conf).getFileSystem(conf); } - + /** - * Runs through the HBase rootdir and creates a reverse lookup map for - * table StoreFile names to the full Path. + * Runs through the HBase rootdir and creates a reverse lookup map for + * table StoreFile names to the full Path. *
    * Example...
    * Key = 3944417774205889744
    @@ -1306,8 +1306,8 @@ final FileSystem fs, final Path hbaseRootDir) throws IOException { Map map = new HashMap(); - - // if this method looks similar to 'getTableFragmentation' that is because + + // if this method looks similar to 'getTableFragmentation' that is because // it was borrowed from it. DirFilter df = new DirFilter(fs); @@ -1338,17 +1338,17 @@ Path sf = sfStatus.getPath(); map.put( sf.getName(), sf); } - + } } } return map; } - + /** * Calls fs.listStatus() and treats FileNotFoundException as non-fatal * This accommodates differences between hadoop versions - * + * * @param fs file system * @param dir directory * @param filter path filter @@ -1381,7 +1381,7 @@ /** * Calls fs.delete() and returns the value returned by the fs.delete() - * + * * @param fs * @param path * @param recursive @@ -1395,7 +1395,7 @@ /** * Calls fs.exists(). Checks if the specified path exists - * + * * @param fs * @param path * @return the value returned by fs.exists() @@ -1407,7 +1407,7 @@ /** * Throw an exception if an action is not permitted by a user on a file. - * + * * @param ugi * the user * @param file @@ -1456,7 +1456,7 @@ /** * Recursive helper to log the state of the FS - * + * * @see #logFileSystemState(FileSystem, Path, Log) */ private static void logFSTree(Log LOG, final FileSystem fs, final Path root, String prefix) Index: hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java =================================================================== --- hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java (revision 1454498) +++ hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java (working copy) @@ -18,23 +18,6 @@ */ package org.apache.hadoop.hbase; -import com.google.protobuf.ByteString; -import com.google.protobuf.InvalidProtocolBufferException; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.exceptions.DeserializationException; -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.hadoop.hbase.protobuf.ProtobufUtil; -import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.BytesBytesPair; -import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ColumnFamilySchema; -import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameStringPair; -import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.TableSchema; -import org.apache.hadoop.hbase.security.User; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.Writables; -import org.apache.hadoop.io.WritableComparable; - import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; @@ -51,10 +34,28 @@ import java.util.TreeSet; import java.util.regex.Matcher; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.exceptions.DeserializationException; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.BytesBytesPair; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ColumnFamilySchema; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameStringPair; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.TableSchema; +import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Writables; +import org.apache.hadoop.io.WritableComparable; + +import com.google.protobuf.ByteString; +import com.google.protobuf.InvalidProtocolBufferException; + /** * HTableDescriptor contains the details about an HBase table such as the descriptors of - * all the column families, is the table a catalog table, -ROOT- or - * .META. , is the table is read only, the maximum size of the memstore, + * all the column families, is the table a catalog table, -ROOT- or + * .META. , if the table is read only, the maximum size of the memstore, * when the region split should occur, coprocessors associated with it etc... */ @InterfaceAudience.Public @@ -76,7 +77,7 @@ private String nameAsString = ""; /** - * A map which holds the metadata information of the table. This metadata + * A map which holds the metadata information of the table. This metadata * includes values like IS_ROOT, IS_META, DEFERRED_LOG_FLUSH, SPLIT_POLICY, * MAX_FILE_SIZE, READONLY, MEMSTORE_FLUSHSIZE etc... */ @@ -93,10 +94,10 @@ public static final String SPLIT_POLICY = "SPLIT_POLICY"; /** - * INTERNAL Used by HBase Shell interface to access this metadata - * attribute which denotes the maximum size of the store file after which + * INTERNAL Used by HBase Shell interface to access this metadata + * attribute which denotes the maximum size of the store file after which * a region split occurs - * + * * @see #getMaxFileSize() */ public static final String MAX_FILESIZE = "MAX_FILESIZE"; @@ -108,9 +109,9 @@ new ImmutableBytesWritable(Bytes.toBytes(OWNER)); /** - * INTERNAL Used by rest interface to access this metadata + * INTERNAL Used by rest interface to access this metadata * attribute which denotes if the table is Read Only - * + * * @see #isReadOnly() */ public static final String READONLY = "READONLY"; @@ -118,10 +119,10 @@ new ImmutableBytesWritable(Bytes.toBytes(READONLY)); /** - * INTERNAL Used by HBase Shell interface to access this metadata - * attribute which represents the maximum size of the memstore after which + * INTERNAL Used by HBase Shell interface to access this metadata + * attribute which represents the maximum size of the memstore after which * its contents are flushed onto the disk - * + * * @see #getMemStoreFlushSize() */ public static final String MEMSTORE_FLUSHSIZE = "MEMSTORE_FLUSHSIZE"; @@ -129,9 +130,9 @@ new ImmutableBytesWritable(Bytes.toBytes(MEMSTORE_FLUSHSIZE)); /** - * INTERNAL Used by rest interface to access this metadata + * INTERNAL Used by rest interface to access this metadata * attribute which denotes if the table is a -ROOT- region or not - * + * * @see #isRootRegion() */ public static final String IS_ROOT = "IS_ROOT"; @@ -139,10 +140,10 @@ new ImmutableBytesWritable(Bytes.toBytes(IS_ROOT)); /** - * INTERNAL Used by rest interface to access this metadata + * INTERNAL Used by rest interface to access this metadata * attribute which denotes if it is a catalog table, either * .META. or -ROOT- - * + * * @see #isMetaRegion() */ public static final String IS_META = "IS_META"; @@ -150,7 +151,7 @@ new ImmutableBytesWritable(Bytes.toBytes(IS_META)); /** - * INTERNAL Used by HBase Shell interface to access this metadata + * INTERNAL Used by HBase Shell interface to access this metadata * attribute which denotes if the deferred log flush option is enabled */ public static final String DEFERRED_LOG_FLUSH = "DEFERRED_LOG_FLUSH"; @@ -176,7 +177,7 @@ public static final boolean DEFAULT_READONLY = false; /** - * Constant that denotes the maximum default size of the memstore after which + * Constant that denotes the maximum default size of the memstore after which * the contents are flushed to the store files */ public static final long DEFAULT_MEMSTORE_FLUSH_SIZE = 1024*1024*128L; @@ -313,9 +314,9 @@ } /* - * Set meta flags on this table. + * Set meta flags on this table. * IS_ROOT_KEY is set if its a -ROOT- table - * IS_META_KEY is set either if its a -ROOT- or a .META. table + * IS_META_KEY is set either if its a -ROOT- or a .META. table * Called by constructors. * @param name */ @@ -327,8 +328,8 @@ /** * Check if the descriptor represents a -ROOT- region. - * - * @return true if this is a -ROOT- region + * + * @return true if this is a -ROOT- region */ public boolean isRootRegion() { if (this.root == null) { @@ -338,11 +339,11 @@ } /** - * INTERNAL Used to denote if the current table represents - * -ROOT- region. This is used internally by the - * HTableDescriptor constructors - * - * @param isRoot true if this is the -ROOT- region + * INTERNAL Used to denote if the current table represents + * -ROOT- region. This is used internally by the + * HTableDescriptor constructors + * + * @param isRoot true if this is the -ROOT- region */ protected void setRootRegion(boolean isRoot) { // TODO: Make the value a boolean rather than String of boolean. @@ -351,10 +352,10 @@ /** * Checks if this table is either -ROOT- or .META. - * region. - * - * @return true if this is either a -ROOT- or .META. - * region + * region. + * + * @return true if this is either a -ROOT- or .META. + * region */ public boolean isMetaRegion() { if (this.meta == null) { @@ -379,31 +380,31 @@ } /** - * INTERNAL Used to denote if the current table represents - * -ROOT- or .META. region. This is used - * internally by the HTableDescriptor constructors - * - * @param isMeta true if its either -ROOT- or - * .META. region + * INTERNAL Used to denote if the current table represents + * -ROOT- or .META. region. This is used + * internally by the HTableDescriptor constructors + * + * @param isMeta true if its either -ROOT- or + * .META. region */ protected void setMetaRegion(boolean isMeta) { setValue(IS_META_KEY, isMeta? TRUE: FALSE); } - /** - * Checks if the table is a .META. table - * + /** + * Checks if the table is a .META. table + * * @return true if table is .META. region. */ public boolean isMetaTable() { return isMetaRegion() && !isRootRegion(); } - + /** - * Checks of the tableName being passed represents either + * Checks of the tableName being passed represents either * -ROOT- or .META. - * - * @return true if a tablesName is either -ROOT- + * + * @return true if a tablesName is either -ROOT- * or .META. */ public static boolean isMetaTable(final byte [] tableName) { @@ -441,7 +442,7 @@ + " conflicted with system reserved words"); } for (int i = 0; i < tableName.length; i++) { - if (Character.isLetterOrDigit(tableName[i]) || tableName[i] == '_' || + if (Character.isLetterOrDigit(tableName[i]) || tableName[i] == '_' || tableName[i] == '-' || tableName[i] == '.') { continue; } @@ -454,7 +455,7 @@ /** * Getter for accessing the metadata associated with the key - * + * * @param key The key. * @return The value. * @see #values @@ -472,7 +473,7 @@ /** * Getter for accessing the metadata associated with the key - * + * * @param key The key. * @return The value. * @see #values @@ -486,7 +487,7 @@ /** * Getter for fetching an unmodifiable {@link #values} map. - * + * * @return unmodifiable map {@link #values}. * @see #values */ @@ -497,7 +498,7 @@ /** * Setter for storing metadata as a (key, value) pair in {@link #values} map - * + * * @param key The key. * @param value The value. * @see #values @@ -526,7 +527,7 @@ /** * Setter for storing metadata as a (key, value) pair in {@link #values} map - * + * * @param key The key. * @param value The value. * @see #values @@ -538,10 +539,10 @@ setValue(Bytes.toBytes(key), Bytes.toBytes(value)); } } - + /** * Remove metadata represented by the key from the {@link #values} map - * + * * @param key Key whose key and value we're to remove from HTableDescriptor * parameters. */ @@ -560,9 +561,9 @@ } /** - * Check if the readOnly flag of the table is set. If the readOnly flag is + * Check if the readOnly flag of the table is set. If the readOnly flag is * set then the contents of the table can only be read from but not modified. - * + * * @return true if all columns in the table should be read only */ public boolean isReadOnly() { @@ -571,9 +572,9 @@ /** * Setting the table as read only sets all the columns in the table as read - * only. By default all tables are modifiable, but if the readOnly flag is + * only. By default all tables are modifiable, but if the readOnly flag is * set to true then the contents of the table can only be read but not modified. - * + * * @param readOnly True if all of the columns in the table should be read * only. */ @@ -582,10 +583,10 @@ } /** - * Check if deferred log edits are enabled on the table. - * + * Check if deferred log edits are enabled on the table. + * * @return true if that deferred log flush is enabled on the table - * + * * @see #setDeferredLogFlush(boolean) */ public synchronized boolean isDeferredLogFlush() { @@ -597,17 +598,17 @@ } /** - * This is used to defer the log edits syncing to the file system. Everytime - * an edit is sent to the server it is first sync'd to the file system by the - * log writer. This sync is an expensive operation and thus can be deferred so + * This is used to defer the log edits syncing to the file system. Everytime + * an edit is sent to the server it is first sync'd to the file system by the + * log writer. This sync is an expensive operation and thus can be deferred so * that the edits are kept in memory for a specified period of time as represented * by hbase.regionserver.optionallogflushinterval and not flushed * for every edit. *

    * NOTE:- This option might result in data loss if the region server crashes - * before these deferred edits in memory are flushed onto the filesystem. + * before these deferred edits in memory are flushed onto the filesystem. *

    - * + * * @param isDeferredLogFlush */ public synchronized void setDeferredLogFlush(final boolean isDeferredLogFlush) { @@ -617,8 +618,8 @@ /** * Get the name of the table as a byte array. - * - * @return name of table + * + * @return name of table */ public byte [] getName() { return name; @@ -626,18 +627,18 @@ /** * Get the name of the table as a String - * - * @return name of table as a String + * + * @return name of table as a String */ public String getNameAsString() { return this.nameAsString; } - + /** - * This get the class associated with the region split policy which + * This get the class associated with the region split policy which * determines when a region split should occur. The class used by * default is defined in {@link org.apache.hadoop.hbase.regionserver.RegionSplitPolicy} - * + * * @return the class name of the region split policy for this table. * If this returns null, the default split policy is used. */ @@ -646,9 +647,9 @@ } /** - * Set the name of the table. - * - * @param name name of table + * Set the name of the table. + * + * @param name name of table */ public void setName(byte[] name) { this.name = name; @@ -656,7 +657,7 @@ setMetaFlags(this.name); } - /** + /** * Returns the maximum size upto which a region can grow to after which a region * split is triggered. The region size is represented by the size of the biggest * store file in that region. @@ -672,19 +673,19 @@ } return -1; } - + /** * Sets the maximum size upto which a region can grow to after which a region - * split is triggered. The region size is represented by the size of the biggest - * store file in that region, i.e. If the biggest store file grows beyond the - * maxFileSize, then the region split is triggered. This defaults to a value of + * split is triggered. The region size is represented by the size of the biggest + * store file in that region, i.e. If the biggest store file grows beyond the + * maxFileSize, then the region split is triggered. This defaults to a value of * 256 MB. *

    - * This is not an absolute value and might vary. Assume that a single row exceeds + * This is not an absolute value and might vary. Assume that a single row exceeds * the maxFileSize then the storeFileSize will be greater than maxFileSize since - * a single row cannot be split across multiple regions + * a single row cannot be split across multiple regions *

    - * + * * @param maxFileSize The maximum file size that a store file can grow to * before a split is triggered. */ @@ -708,9 +709,9 @@ } /** - * Represents the maximum size of the memstore after which the contents of the + * Represents the maximum size of the memstore after which the contents of the * memstore are flushed to the filesystem. This defaults to a size of 64 MB. - * + * * @param memstoreFlushSize memory cache flush size for each hregion */ public void setMemStoreFlushSize(long memstoreFlushSize) { @@ -857,12 +858,12 @@ } /** - * Compare the contents of the descriptor with another one passed as a parameter. + * Compare the contents of the descriptor with another one passed as a parameter. * Checks if the obj passed is an instance of HTableDescriptor, if yes then the * contents of the descriptors are compared. - * + * * @return true if the contents of the the two descriptors exactly match - * + * * @see java.lang.Object#equals(java.lang.Object) */ @Override @@ -897,7 +898,7 @@ } /** - * INTERNAL This method is a part of {@link WritableComparable} interface + * INTERNAL This method is a part of {@link WritableComparable} interface * and is used for de-serialization of the HTableDescriptor over RPC * @deprecated Writables are going away. Use pb {@link #parseFrom(byte[])} instead. */ @@ -944,7 +945,7 @@ } /** - * INTERNAL This method is a part of {@link WritableComparable} interface + * INTERNAL This method is a part of {@link WritableComparable} interface * and is used for serialization of the HTableDescriptor over RPC * @deprecated Writables are going away. * Use {@link com.google.protobuf.MessageLite#toByteArray} instead. @@ -980,9 +981,9 @@ /** * Compares the descriptor with another descriptor which is passed as a parameter. * This compares the content of the two descriptors and not the reference. - * - * @return 0 if the contents of the descriptors are exactly matching, - * 1 if there is a mismatch in the contents + * + * @return 0 if the contents of the descriptors are exactly matching, + * 1 if there is a mismatch in the contents */ @Override public int compareTo(final HTableDescriptor other) { @@ -1022,59 +1023,59 @@ } /** - * Returns an unmodifiable collection of all the {@link HColumnDescriptor} + * Returns an unmodifiable collection of all the {@link HColumnDescriptor} * of all the column families of the table. - * + * * @return Immutable collection of {@link HColumnDescriptor} of all the - * column families. + * column families. */ public Collection getFamilies() { return Collections.unmodifiableCollection(this.families.values()); } /** - * Returns all the column family names of the current table. The map of - * HTableDescriptor contains mapping of family name to HColumnDescriptors. - * This returns all the keys of the family map which represents the column - * family names of the table. - * + * Returns all the column family names of the current table. The map of + * HTableDescriptor contains mapping of family name to HColumnDescriptors. + * This returns all the keys of the family map which represents the column + * family names of the table. + * * @return Immutable sorted set of the keys of the families. */ public Set getFamiliesKeys() { return Collections.unmodifiableSet(this.families.keySet()); } - /** - * Returns an array all the {@link HColumnDescriptor} of the column families + /** + * Returns an array all the {@link HColumnDescriptor} of the column families * of the table. - * - * @return Array of all the HColumnDescriptors of the current table - * + * + * @return Array of all the HColumnDescriptors of the current table + * * @see #getFamilies() */ public HColumnDescriptor[] getColumnFamilies() { Collection hColumnDescriptors = getFamilies(); return hColumnDescriptors.toArray(new HColumnDescriptor[hColumnDescriptors.size()]); } - + /** - * Returns the HColumnDescriptor for a specific column family with name as + * Returns the HColumnDescriptor for a specific column family with name as * specified by the parameter column. - * - * @param column Column family name + * + * @param column Column family name * @return Column descriptor for the passed family name or the family on * passed in column. */ public HColumnDescriptor getFamily(final byte [] column) { return this.families.get(column); } - + /** - * Removes the HColumnDescriptor with name specified by the parameter column + * Removes the HColumnDescriptor with name specified by the parameter column * from the table descriptor - * + * * @param column Name of the column family to be removed. * @return Column descriptor for the passed family name or the family on * passed in column. @@ -1082,8 +1083,8 @@ public HColumnDescriptor removeFamily(final byte [] column) { return this.families.remove(column); } - + /** * Add a table coprocessor to this table. The coprocessor * type must be {@link org.apache.hadoop.hbase.coprocessor.RegionObserver} @@ -1098,7 +1099,7 @@ addCoprocessor(className, null, Coprocessor.PRIORITY_USER, null); } - + /** * Add a table coprocessor to this table. The coprocessor * type must be {@link org.apache.hadoop.hbase.coprocessor.RegionObserver} @@ -1251,11 +1252,11 @@ if (match != null) remove(match); } - + /** - * Returns the {@link Path} object representing the table directory under - * path rootdir - * + * Returns the {@link Path} object representing the table directory under + * path rootdir + * * @param rootdir qualified path of HBase root directory * @param tableName name of table * @return {@link Path} for table @@ -1288,6 +1289,18 @@ .setScope(HConstants.REPLICATION_SCOPE_LOCAL) }); + static { + try { + META_TABLEDESC.addCoprocessor( + "org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint", + null, Coprocessor.PRIORITY_SYSTEM, null); + } catch (IOException ex) { + //LOG.warn("exception in loading coprocessor for the META table"); + throw new RuntimeException(ex); + } + } + + @Deprecated public void setOwner(User owner) { setOwnerString(owner != null ? owner.getShortName() : null); @@ -1326,7 +1339,7 @@ * @param bytes A pb serialized {@link HTableDescriptor} instance with pb magic prefix * @return An instance of {@link HTableDescriptor} made from bytes * @throws DeserializationException - * @throws IOException + * @throws IOException * @see #toByteArray() */ public static HTableDescriptor parseFrom(final byte [] bytes) Index: hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java =================================================================== --- hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java (revision 1454498) +++ hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java (working copy) @@ -37,7 +37,6 @@ import java.util.List; import java.util.NavigableMap; import java.util.TreeMap; -import java.util.TreeSet; /** * Scanner class that contains the .META. table scanning logic @@ -49,7 +48,7 @@ * *

    Note that during concurrent region splits, the scanner might not see * META changes across rows (for parent and daughter entries) consistently. - * see HBASE-5986, and {@link BlockingMetaScannerVisitor} for details.

    + * see HBASE-5986, and {@link DefaultMetaScannerVisitor} for details.

    */ @InterfaceAudience.Private public class MetaScanner { @@ -276,7 +275,7 @@ public static List listAllRegions(Configuration conf, final boolean offlined) throws IOException { final List regions = new ArrayList(); - MetaScannerVisitor visitor = new BlockingMetaScannerVisitor(conf) { + MetaScannerVisitor visitor = new DefaultMetaScannerVisitor(conf) { @Override public boolean processRowInternal(Result result) throws IOException { if (result == null || result.isEmpty()) { @@ -316,10 +315,7 @@ public boolean processRowInternal(Result rowResult) throws IOException { HRegionInfo info = getHRegionInfo(rowResult); ServerName serverName = HRegionInfo.getServerName(rowResult); - - if (!(info.isOffline() || info.isSplit())) { - regions.put(new UnmodifyableHRegionInfo(info), serverName); - } + regions.put(new UnmodifyableHRegionInfo(info), serverName); return true; } }; @@ -355,108 +351,30 @@ * does not guarantee ordered traversal of meta entries, and can block until the * META entries for daughters are available during splits. */ - public static abstract class BlockingMetaScannerVisitor + public static abstract class DefaultMetaScannerVisitor extends MetaScannerVisitorBase { - private static final int DEFAULT_BLOCKING_TIMEOUT = 10000; - private Configuration conf; - private TreeSet daughterRegions = new TreeSet(Bytes.BYTES_COMPARATOR); - private int blockingTimeout; - private HTable metaTable; + protected Configuration conf; - public BlockingMetaScannerVisitor(Configuration conf) { + public DefaultMetaScannerVisitor(Configuration conf) { this.conf = conf; - this.blockingTimeout = conf.getInt(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, - DEFAULT_BLOCKING_TIMEOUT); } public abstract boolean processRowInternal(Result rowResult) throws IOException; @Override - public void close() throws IOException { - super.close(); - if (metaTable != null) { - metaTable.close(); - metaTable = null; - } - } - - public HTable getMetaTable() throws IOException { - if (metaTable == null) { - metaTable = new HTable(conf, HConstants.META_TABLE_NAME); - } - return metaTable; - } - - @Override public boolean processRow(Result rowResult) throws IOException { HRegionInfo info = getHRegionInfo(rowResult); if (info == null) { return true; } - if (daughterRegions.remove(info.getRegionName())) { - return true; //we have already processed this row + //skip over offline and split regions + if (!(info.isOffline() || info.isSplit())) { + return processRowInternal(rowResult); } - - if (info.isSplitParent()) { - /* we have found a parent region which was split. We have to ensure that it's daughters are - * seen by this scanner as well, so we block until they are added to the META table. Even - * though we are waiting for META entries, ACID semantics in HBase indicates that this - * scanner might not see the new rows. So we manually query the daughter rows */ - PairOfSameType daughters = HRegionInfo.getDaughterRegions(rowResult); - HRegionInfo splitA = daughters.getFirst(); - HRegionInfo splitB = daughters.getSecond(); - - HTable metaTable = getMetaTable(); - long start = System.currentTimeMillis(); - Result resultA = getRegionResultBlocking(metaTable, blockingTimeout, - splitA.getRegionName()); - if (resultA != null) { - processRow(resultA); - daughterRegions.add(splitA.getRegionName()); - } else { - throw new RegionOfflineException("Split daughter region " + - splitA.getRegionNameAsString() + " cannot be found in META."); - } - long rem = blockingTimeout - (System.currentTimeMillis() - start); - - Result resultB = getRegionResultBlocking(metaTable, rem, - splitB.getRegionName()); - if (resultB != null) { - processRow(resultB); - daughterRegions.add(splitB.getRegionName()); - } else { - throw new RegionOfflineException("Split daughter region " + - splitB.getRegionNameAsString() + " cannot be found in META."); - } - } - - return processRowInternal(rowResult); + return true; } - - private Result getRegionResultBlocking(HTable metaTable, long timeout, byte[] regionName) - throws IOException { - if (LOG.isDebugEnabled()) { - LOG.debug("blocking until region is in META: " + Bytes.toStringBinary(regionName)); - } - long start = System.currentTimeMillis(); - while (System.currentTimeMillis() - start < timeout) { - Get get = new Get(regionName); - Result result = metaTable.get(get); - HRegionInfo info = getHRegionInfo(result); - if (info != null) { - return result; - } - try { - Thread.sleep(10); - } catch (InterruptedException ex) { - Thread.currentThread().interrupt(); - break; - } - } - return null; - } } /** @@ -465,7 +383,7 @@ * does not guarantee ordered traversal of meta entries, and can block until the * META entries for daughters are available during splits. */ - public static abstract class TableMetaScannerVisitor extends BlockingMetaScannerVisitor { + public static abstract class TableMetaScannerVisitor extends DefaultMetaScannerVisitor { private byte[] tableName; public TableMetaScannerVisitor(Configuration conf, byte[] tableName) {