diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java index e236d99..bc4c74c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java @@ -573,7 +573,15 @@ public class MetaEditor extends MetaReader { return p; } - private static Put addLocation(final Put p, final ServerName sn, long openSeqNum, int replicaId){ + /** + * Convenience method. Add replica related columns to a Put request. + * @param p + * @param sn + * @param openSeqNum + * @param replicaId + * @return + */ + public static Put addLocation(final Put p, final ServerName sn, long openSeqNum, int replicaId){ p.addImmutable(HConstants.CATALOG_FAMILY, MetaReader.getServerColumn(replicaId), Bytes.toBytes(sn.getHostAndPort())); p.addImmutable(HConstants.CATALOG_FAMILY, MetaReader.getStartCodeColumn(replicaId), diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java index 40d1ce2..0815538 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java @@ -68,10 +68,12 @@ import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.MasterNotRunningException; +import org.apache.hadoop.hbase.RegionLocations; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.ZooKeeperConnectionException; import org.apache.hadoop.hbase.catalog.MetaEditor; +import org.apache.hadoop.hbase.catalog.MetaReader; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.HBaseAdmin; @@ -1668,6 +1670,19 @@ public class HBaseFsck extends Configured { try { HBaseFsckRepair.closeRegionSilentlyAndWait(admin, rse.hsa, rse.hri); offline(rse.hri.getRegionName()); + // also undeploy replicas if needed + // TODO: needs to address this in a better way + int replicationCount = admin.getTableDescriptor(rse.hri.getTable()).getRegionReplication(); + for (int i = 1; i < replicationCount; i++) { + HRegionInfo hri = RegionReplicaUtil.getRegionInfoForReplica(rse.hri, i); + HbckInfo hiReplica = regionInfoMap.get(hri.getEncodedName()); + for (OnlineEntry hseReplica : hiReplica.deployedEntries) { + LOG.info("Closing region " + hseReplica.hri + " on server " + hseReplica.hsa); + HBaseFsckRepair.closeRegionSilentlyAndWait(admin, hseReplica.hsa, + hseReplica.hri); + offline(hseReplica.hri.getRegionName()); + } + } } catch (IOException ioe) { LOG.warn("Got exception when attempting to offline region " + Bytes.toString(rse.hri.getRegionName()), ioe); @@ -1687,7 +1702,7 @@ public class HBaseFsck extends Configured { * the offline ipc call exposed on the master (<0.90.5, <0.92.0) a master * restart or failover may be required. */ - private void closeRegion(HbckInfo hi) throws IOException, InterruptedException { + private void closeRegion(HbckInfo hi, int numReplicas) throws IOException, InterruptedException { if (hi.metaEntry == null && hi.hdfsEntry == null) { undeployRegions(hi); return; @@ -1696,29 +1711,35 @@ public class HBaseFsck extends Configured { // get assignment info and hregioninfo from meta. Get get = new Get(hi.getRegionName()); get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER); - get.addColumn(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER); - get.addColumn(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER); - Result r = meta.get(get); - ServerName serverName = HRegionInfo.getServerName(r); - if (serverName == null) { - errors.reportError("Unable to close region " - + hi.getRegionNameAsString() + " because meta does not " - + "have handle to reach it."); - return; + for (int i = 0; i < numReplicas; i++) { + get.addColumn(HConstants.CATALOG_FAMILY, MetaReader.getServerColumn(i)); + get.addColumn(HConstants.CATALOG_FAMILY, MetaReader.getStartCodeColumn(i)); } + Result r = meta.get(get); + RegionLocations rl = MetaReader.getRegionLocations(r); + if (rl == null) return; + for (HRegionLocation h : rl.getRegionLocations()) { + ServerName serverName = h.getServerName(); + if (serverName == null) { + errors.reportError("Unable to close region " + + hi.getRegionNameAsString() + " because meta does not " + + "have handle to reach it."); + continue; + } - HRegionInfo hri = HRegionInfo.getHRegionInfo(r); - if (hri == null) { - LOG.warn("Unable to close region " + hi.getRegionNameAsString() - + " because hbase:meta had invalid or missing " - + HConstants.CATALOG_FAMILY_STR + ":" - + Bytes.toString(HConstants.REGIONINFO_QUALIFIER) - + " qualifier value."); - return; - } + HRegionInfo hri = h.getRegionInfo(); + if (hri == null) { + LOG.warn("Unable to close region " + hi.getRegionNameAsString() + + " because hbase:meta had invalid or missing " + + HConstants.CATALOG_FAMILY_STR + ":" + + Bytes.toString(HConstants.REGIONINFO_QUALIFIER) + + " qualifier value."); + continue; + } - // close the region -- close files and remove assignment - HBaseFsckRepair.closeRegionSilentlyAndWait(admin, serverName, hri); + // close the region -- close files and remove assignment + HBaseFsckRepair.closeRegionSilentlyAndWait(admin, serverName, hri); + } } private void tryAssignmentRepair(HbckInfo hbi, String msg) throws IOException, @@ -1734,6 +1755,14 @@ public class HBaseFsck extends Configured { } HBaseFsckRepair.fixUnassigned(admin, hri); HBaseFsckRepair.waitUntilAssigned(admin, hri); + + // also assign replicas if needed + int replicationCount = admin.getTableDescriptor(hri.getTable()).getRegionReplication(); + for (int i = 1; i < replicationCount; i++) { + hri = RegionReplicaUtil.getRegionInfoForReplica(hri, i); + HBaseFsckRepair.fixUnassigned(admin, hri); + HBaseFsckRepair.waitUntilAssigned(admin, hri); + } } } @@ -1785,7 +1814,6 @@ public class HBaseFsck extends Configured { if (shouldFixAssignments()) { undeployRegions(hbi); } - } else if (!inMeta && inHdfs && !isDeployed) { if (hbi.isMerged()) { // This region has already been merged, the remaining hdfs file will be @@ -1808,8 +1836,10 @@ public class HBaseFsck extends Configured { } LOG.info("Patching hbase:meta with .regioninfo: " + hbi.getHdfsHRI()); - HBaseFsckRepair.fixMetaHoleOnline(getConf(), hbi.getHdfsHRI()); - + //HBaseFsckRepair.fixMetaHoleOnline(getConf(), hbi.getHdfsHRI()); + int numReplicas = admin.getTableDescriptor(hbi.getTableName()).getRegionReplication(); + HBaseFsckRepair.fixMetaHoleOnlineAndAddReplicas(getConf(), hbi.getHdfsHRI(), + admin.getClusterStatus().getServers(), numReplicas); tryAssignmentRepair(hbi, "Trying to reassign region..."); } @@ -1866,7 +1896,7 @@ public class HBaseFsck extends Configured { // these problems from META. if (shouldFixAssignments()) { errors.print("Trying to fix unassigned region..."); - closeRegion(hbi);// Close region will cause RS to abort. + closeRegion(hbi, admin.getTableDescriptor(hbi.getTableName()).getRegionReplication()); } if (shouldFixMeta()) { // wait for it to complete @@ -2382,7 +2412,7 @@ public class HBaseFsck extends Configured { debugLsr(hi.getHdfsRegionDir()); try { LOG.info("[" + thread + "] Closing region: " + hi); - closeRegion(hi); + closeRegion(hi, 1); //TODO: does it need the replica handling? } catch (IOException ioe) { LOG.warn("[" + thread + "] Was unable to close region " + hi + ". Just continuing... ", ioe); @@ -2444,7 +2474,7 @@ public class HBaseFsck extends Configured { for (HbckInfo regionToSideline: regionsToSideline) { try { LOG.info("Closing region: " + regionToSideline); - closeRegion(regionToSideline); + closeRegion(regionToSideline, 1); //TODO: does it need replica handling } catch (IOException ioe) { LOG.warn("Was unable to close region " + regionToSideline + ". Just continuing... ", ioe); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java index 727c0d4..7470e05 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java @@ -19,8 +19,10 @@ package org.apache.hadoop.hbase.util; import java.io.IOException; +import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.Random; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -36,6 +38,7 @@ import org.apache.hadoop.hbase.catalog.MetaEditor; import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.client.HConnection; import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.master.RegionState; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService; @@ -181,6 +184,29 @@ public class HBaseFsckRepair { } /** + * Puts the specified HRegionInfo into META with replica related columns + */ + public static void fixMetaHoleOnlineAndAddReplicas(Configuration conf, + HRegionInfo hri, Collection servers, int numReplicas) throws IOException { + HTable meta = new HTable(conf, TableName.META_TABLE_NAME); + Put put = MetaEditor.makePutFromRegionInfo(hri); + if (numReplicas > 1) { + Random r = new Random(); + ServerName[] serversArr = servers.toArray(new ServerName[0]); + for (int i = 1; i < numReplicas; i++) { + ServerName sn = serversArr[r.nextInt(serversArr.length)]; + // the column added here is just to make sure the master is able to + // see the additional replicas when it is asked to assign. The + // final value of these columns will be different and will be updated + // by the actual regionservers that start hosting the respective replicas + MetaEditor.addLocation(put, sn, sn.getStartcode(), i); + } + } + meta.put(put); + meta.close(); + } + + /** * Creates, flushes, and closes a new region. */ public static HRegion createHDFSRegionDir(Configuration conf, diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java index efb6ff3..ff280b9 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java @@ -30,12 +30,16 @@ import static org.junit.Assert.fail; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.NavigableMap; +import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledThreadPoolExecutor; @@ -77,6 +81,7 @@ import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.io.hfile.TestHFile; import org.apache.hadoop.hbase.master.AssignmentManager; import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.master.RegionState; import org.apache.hadoop.hbase.master.RegionStates; import org.apache.hadoop.hbase.master.TableLockManager; import org.apache.hadoop.hbase.master.TableLockManager.TableLock; @@ -1110,6 +1115,83 @@ public class TestHBaseFsck { } /** + * This creates and fixes a bad table with a region that is in meta but has + * no deployment or data hdfs. The table has region_replication set to 2. + */ + @Test + public void testNotInHdfsWithReplicas() throws Exception { + TableName table = + TableName.valueOf("tableNotInHdfs"); + HBaseAdmin admin = new HBaseAdmin(conf); + try { + HRegionInfo[] oldHris = new HRegionInfo[2]; + setupTableWithRegionReplica(table, 2); + assertEquals(ROWKEYS.length, countRows()); + NavigableMap map = MetaScanner.allTableRegions(conf, null, + tbl.getName(), false); + int i = 0; + // store the HRIs of the regions we will mess up + for (Map.Entry m : map.entrySet()) { + if (m.getKey().getStartKey().length > 0 && + m.getKey().getStartKey()[0] == Bytes.toBytes("B")[0]) { + LOG.debug("Initially server hosting " + m.getKey() + " is " + m.getValue()); + oldHris[i++] = m.getKey(); + } + } + // make sure data in regions + TEST_UTIL.getHBaseAdmin().flush(table.getName()); + + // Mess it up by leaving a hole in the hdfs data + deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"), + Bytes.toBytes("C"), false, false, true); // don't rm meta + + HBaseFsck hbck = doFsck(conf, false); + assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.NOT_IN_HDFS}); + + // fix hole + doFsck(conf, true); + + // check that hole fixed + assertNoErrors(doFsck(conf,false)); + assertEquals(ROWKEYS.length - 2, countRows()); + + // the following code checks whether the old primary/secondary has + // been unassigned and the new primary/secondary has been assigned + i = 0; + HRegionInfo[] newHris = new HRegionInfo[2]; + // get all table's regions from meta + map = MetaScanner.allTableRegions(conf, null, tbl.getName(), false); + // get the HRIs of the new regions (hbck created new regions for fixing the hdfs mess-up) + for (Map.Entry m : map.entrySet()) { + if (m.getKey().getStartKey().length > 0 && + m.getKey().getStartKey()[0] == Bytes.toBytes("B")[0]) { + LOG.debug("Server hosting " + m.getKey() + " is " + m.getValue()); + newHris[i++] = m.getKey(); + } + } + // get all the online regions in the regionservers + Collection servers = admin.getClusterStatus().getServers(); + Set onlineRegions = new HashSet(); + for (ServerName s : servers) { + List list = admin.getOnlineRegions(s); + onlineRegions.addAll(list); + for (HRegionInfo h : list) { + LOG.debug("ONLINE region " + h); + } + } + // the new HRIs must be a subset of the online regions + assertTrue(onlineRegions.containsAll(Arrays.asList(newHris))); + // the old HRIs must not be part of the set (removeAll would return false if + // the set didn't change) + assertFalse(onlineRegions.removeAll(Arrays.asList(oldHris))); + } finally { + deleteTable(table); + admin.close(); + } + } + + + /** * This creates entries in hbase:meta with no hdfs data. This should cleanly * remove the table. */