From 1464d1740220ff1fc40c019566f44b8997cf1250 Mon Sep 17 00:00:00 2001 From: Michael Stack Date: Fri, 16 Mar 2018 20:36:57 -0700 Subject: [PATCH] HBASE-20169 NPE when calling HBTU.shutdownMiniCluster Adds a prepare step to RecoverMetaProcedure in which we test for cluster up and master being up. If not up, we fail the run. M hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MoveRegionProcedure.java Bug fix. Set proper start step. M hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java M hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ChunkCreator.java Minor log cleanup. M hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RecoverMetaProcedure.java Add pepare step. M hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerMetrics.java Debug for the failing test.... --- .../src/main/protobuf/MasterProcedure.proto | 1 + .../hbase/master/assignment/MoveRegionProcedure.java | 3 ++- .../hadoop/hbase/master/cleaner/HFileCleaner.java | 6 +++--- .../hbase/master/procedure/RecoverMetaProcedure.java | 17 ++++++++++++++++- .../apache/hadoop/hbase/regionserver/ChunkCreator.java | 2 +- .../hbase/master/TestAssignmentManagerMetrics.java | 1 + 6 files changed, 24 insertions(+), 6 deletions(-) diff --git a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto index 9666c25846..fa6fa757a8 100644 --- a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto +++ b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto @@ -307,6 +307,7 @@ enum ServerCrashState { } enum RecoverMetaState { + RECOVER_META_PREPARE = 0; RECOVER_META_SPLIT_LOGS = 1; RECOVER_META_ASSIGN_REGIONS = 2; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MoveRegionProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MoveRegionProcedure.java index c52af3dee8..bdc545f719 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MoveRegionProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MoveRegionProcedure.java @@ -83,6 +83,7 @@ public class MoveRegionProcedure extends AbstractStateMachineRegionProcedure { break; } if (task != null) { - LOG.debug("Removing: {} from archive", task.filePath); + LOG.debug("Removing {}", task.filePath); boolean succeed; try { succeed = this.fs.delete(task.filePath, false); } catch (IOException e) { - LOG.warn("Failed to delete file {}", task.filePath, e); + LOG.warn("Failed to delete {}", task.filePath, e); succeed = false; } task.setResult(succeed); @@ -250,7 +250,7 @@ public class HFileCleaner extends CleanerChore { } } } finally { - LOG.debug("Exit thread: {}", Thread.currentThread()); + LOG.debug("Exit {}", Thread.currentThread()); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RecoverMetaProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RecoverMetaProcedure.java index 301cd181f4..59116e7a5d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RecoverMetaProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RecoverMetaProcedure.java @@ -21,6 +21,7 @@ package org.apache.hadoop.hbase.master.procedure; import java.io.IOException; import java.util.Set; +import org.apache.hadoop.hbase.HBaseIOException; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.RegionInfo; @@ -104,6 +105,20 @@ public class RecoverMetaProcedure try { switch (state) { + case RECOVER_META_PREPARE: + if (!master.isClusterUp()) { + String msg = "Cluster not up!"; + LOG.warn(msg); + throw new HBaseIOException(msg); + } + if (master.isStopping() || master.isStopped()) { + String msg = "Master stopping=" + master.isStopping() + ", stopped=" + + master.isStopped(); + LOG.warn(msg); + throw new HBaseIOException(msg); + } + setNextState(RecoverMetaState.RECOVER_META_SPLIT_LOGS); + break; case RECOVER_META_SPLIT_LOGS: LOG.info("Start " + this); if (shouldSplitWal) { @@ -202,7 +217,7 @@ public class RecoverMetaProcedure @Override protected MasterProcedureProtos.RecoverMetaState getInitialState() { - return RecoverMetaState.RECOVER_META_SPLIT_LOGS; + return RecoverMetaState.RECOVER_META_PREPARE; } @Override diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ChunkCreator.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ChunkCreator.java index 5dc8e48596..eea1ecdd59 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ChunkCreator.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ChunkCreator.java @@ -424,7 +424,7 @@ public class ChunkCreator { long created = chunkCount.get(); long reused = reusedChunkCount.sum(); long total = created + reused; - LOG.debug("{} Stats (chunk size={}): current pool size={}, created chunk count={}, " + + LOG.debug("{} stats (chunk size={}): current pool size={}, created chunk count={}, " + "reused chunk count={}, reuseRatio={}", label, chunkSize, reclaimedChunks.size(), created, reused, (total == 0? "0": StringUtils.formatPercent((float)reused/(float)total,2))); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerMetrics.java index 4aa97d3356..87f6fa4e25 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerMetrics.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerMetrics.java @@ -98,6 +98,7 @@ public class TestAssignmentManagerMetrics { @AfterClass public static void after() throws Exception { + LOG.info("AFTER {} <= IS THIS NULL?", TEST_UTIL); TEST_UTIL.shutdownMiniCluster(); } -- 2.11.0 (Apple Git-81)