diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentVerificationReport.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentVerificationReport.java index b84014a..44c4e42 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentVerificationReport.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentVerificationReport.java @@ -582,6 +582,48 @@ public class AssignmentVerificationReport { System.out.println("=============================="); } + /** + * Return the unassigned regions + * @return unassigned regions + */ + List getUnassignedRegions() { + return unAssignedRegionsList; + } + + /** + * Return the regions without favored nodes + * @return regions without favored nodes + */ + List getRegionsWithoutValidFavoredNodes() { + return regionsWithoutValidFavoredNodes; + } + + /** + * Return the regions not assigned to its favored nodes + * @return regions not assigned to its favored nodes + */ + List getNonFavoredAssignedRegions() { + return nonFavoredAssignedRegionList; + } + + /** + * Return the number of regions assigned to their favored nodes + * @return number of regions assigned to their favored nodes + */ + int getTotalFavoredAssignments() { + return totalFavoredAssignments; + } + + /** + * Return the number of regions based on the position (primary/secondary/ + * tertiary) assigned to their favored nodes + * @param position + * @return the number of regions + */ + int getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position position) { + return favoredNodes[position.ordinal()]; + } + private void printHServerAddressSet(Set serverSet) { if (serverSet == null) { return ; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionPlacementMaintainer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionPlacementMaintainer.java index 9e42ffe..a8831f3 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionPlacementMaintainer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionPlacementMaintainer.java @@ -148,10 +148,13 @@ public class RegionPlacementMaintainer { } /** - * Verify the region placement is consistent with the assignment plan; + * Verify the region placement is consistent with the assignment plan + * @param isDetailMode + * @return reports * @throws IOException */ - public void verifyRegionPlacement(boolean isDetailMode) throws IOException { + public List verifyRegionPlacement(boolean isDetailMode) + throws IOException { System.out.println("Start to verify the region assignment and " + "generate the verification report"); // Get the region assignment snapshot @@ -165,6 +168,7 @@ public class RegionPlacementMaintainer { if (this.enforceLocality == true) { regionLocalityMap = FSUtils.getRegionDegreeLocalityMappingFromFS(conf); } + List reports = new ArrayList(); // Iterate all the tables to fill up the verification report for (TableName table : tables) { if (!this.targetTableSet.isEmpty() && @@ -174,7 +178,9 @@ public class RegionPlacementMaintainer { AssignmentVerificationReport report = new AssignmentVerificationReport(); report.fillUp(table, snapshot, regionLocalityMap); report.print(isDetailMode); + reports.add(report); } + return reports; } /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/FavoredNodesPlan.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/FavoredNodesPlan.java index a430390..d87c0c9 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/FavoredNodesPlan.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/FavoredNodesPlan.java @@ -87,7 +87,7 @@ public class FavoredNodesPlan { return null; } for (Position p : Position.values()) { - if (favoredNodes.get(p.ordinal()).equals(server)) { + if (ServerName.isSameHostnameAndPort(favoredNodes.get(p.ordinal()),server)) { return p; } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSRegionScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSRegionScanner.java index 4c0cc46..7506bec 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSRegionScanner.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSRegionScanner.java @@ -30,6 +30,7 @@ import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HConstants; /** * Thread that walks over the filesystem, and computes the mappings @@ -89,7 +90,8 @@ class FSRegionScanner implements Runnable { // skip because this is not a CF directory continue; } - if (cfStatus.getPath().getName().startsWith(".")) { + if (cfStatus.getPath().getName().startsWith(".") || + HConstants.HBASE_NON_USER_TABLE_DIRS.contains(cfStatus.getPath().getName())) { continue; } FileStatus[] storeFileLists = fs.listStatus(cfStatus.getPath()); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java index d787a0d..7aea051 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java @@ -1785,9 +1785,9 @@ public abstract class FSUtils { Path queryPath; // The table files are in ${hbase.rootdir}/data///* if (null == desiredTable) { - queryPath = new Path(new Path(rootPath, HConstants.BASE_NAMESPACE_DIR), "/*/*/"); + queryPath = new Path(new Path(rootPath, HConstants.BASE_NAMESPACE_DIR).toString() + "/*/*/*/"); } else { - queryPath = new Path(FSUtils.getTableDir(rootPath, TableName.valueOf(desiredTable)), "/*/"); + queryPath = new Path(FSUtils.getTableDir(rootPath, TableName.valueOf(desiredTable)).toString() + "/*/"); } // reject all paths that are not appropriate @@ -1805,11 +1805,6 @@ public abstract class FSUtils { return false; } - // not part of a table? - if (!parent.getName().equals(TableName.META_TABLE_NAME.getQualifierAsString())) { - return false; - } - String regionName = path.getName(); if (null == regionName) { return false; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement.java index f826087..0fc9e40 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement.java @@ -210,7 +210,8 @@ public class TestRegionPlacement { @Test public void testRegionPlacement() throws Exception { - byte[] table = Bytes.toBytes("testRegionAssignment"); + String tableStr = "testRegionAssignment"; + byte[] table = Bytes.toBytes(tableStr); // Create a table with REGION_NUM regions. createTable(table, REGION_NUM); @@ -249,8 +250,36 @@ public class TestRegionPlacement { rp.updateAssignmentPlan(shuffledPlan); verifyRegionAssignment(shuffledPlan, REGION_NUM, REGION_NUM); + + // also verify that the AssignmentVerificationReport has the correct information + RegionPlacementMaintainer rp = new RegionPlacementMaintainer(TEST_UTIL.getConfiguration()); + // we are interested in only one table (and hence one report) + rp.setTargetTableName(new String[]{tableStr}); + List reports = rp.verifyRegionPlacement(false); + AssignmentVerificationReport report = reports.get(0); + assertTrue(report.getRegionsWithoutValidFavoredNodes().size() == 0); + assertTrue(report.getNonFavoredAssignedRegions().size() == 0); + assertTrue(report.getTotalFavoredAssignments() >= REGION_NUM); + assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.PRIMARY) != 0); + assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) == 0); + assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY) == 0); + assertTrue(report.getUnassignedRegions().size() == 0); + // Check when a RS stops, the regions get assigned to their secondary/tertiary killRandomServerAndVerifyAssignment(); + + // also verify that the AssignmentVerificationReport has the correct information + reports = rp.verifyRegionPlacement(false); + report = reports.get(0); + assertTrue(report.getRegionsWithoutValidFavoredNodes().size() == 0); + assertTrue(report.getNonFavoredAssignedRegions().size() == 0); + assertTrue(report.getTotalFavoredAssignments() >= REGION_NUM); + assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.PRIMARY) > 0 && + (report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) > 0 + || report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY) > 0)); + assertTrue((report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.PRIMARY) + + report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) + + report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY)) == REGION_NUM); RegionPlacementMaintainer.printAssignmentPlan(currentPlan); }