diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java index 4a4a5e7b5f..7561df6c40 100644 --- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java +++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java @@ -335,7 +335,7 @@ public class ProcedureExecutor { int corruptedCount = 0; while (procIter.hasNext()) { Procedure proc = procIter.next(); - LOG.error("Corrupt " + proc); + LOG.error("Corrupt proc " + proc); corruptedCount++; } if (abortOnCorruption && corruptedCount > 0) { diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.java index 7d430d66b6..178622ee3f 100644 --- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.java +++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.java @@ -29,7 +29,6 @@ import org.apache.hadoop.hbase.procedure2.Procedure; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos; /** @@ -42,7 +41,6 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos; public class ProcedureStoreTracker { private static final Logger LOG = LoggerFactory.getLogger(ProcedureStoreTracker.class); - // Key is procedure id corresponding to first bit of the bitmap. private final TreeMap map = new TreeMap<>(); @@ -71,6 +69,7 @@ public class ProcedureStoreTracker { for (ProcedureProtos.ProcedureStoreTracker.TrackerNode protoNode: trackerProtoBuf.getNodeList()) { final BitSetNode node = new BitSetNode(protoNode); map.put(node.getStart(), node); + LOG.debug("storing " + node + " for " + node.getStart()); } } diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFile.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFile.java index 16767446e1..9c566ecd5f 100644 --- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFile.java +++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFile.java @@ -76,6 +76,8 @@ public class ProcedureWALFile implements Comparable { public void open() throws IOException { if (stream == null) { stream = fs.open(logFile); + FileStatus fileStatus = fs.getFileStatus(logFile); + LOG.info("Opening {} length={}", logFile, fileStatus.getLen()); } if (header == null) { @@ -114,6 +116,8 @@ public class ProcedureWALFile implements Comparable { if (stream == null) return; try { stream.close(); + FileStatus fileStatus = fs.getFileStatus(logFile); + LOG.info("Closed {} length={}", logFile, fileStatus.getLen()); } catch (IOException e) { LOG.warn("unable to close the wal file: " + logFile, e); } finally { diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.java index e8a6d97a31..b8e08eb570 100644 --- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.java +++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.java @@ -185,6 +185,7 @@ public class WALProcedureStore extends ProcedureStoreBase { private LinkedTransferQueue slotsCache = null; private Set corruptedLogs = null; private FSDataOutputStream stream = null; + private Path logFile = null; private int runningProcCount = 1; private long flushLogId = 0; private int syncMaxSlot = 1; @@ -929,6 +930,9 @@ public class WALProcedureStore extends ProcedureStoreBase { syncStream(stream); sendPostSyncSignal(); + if (LOG.isDebugEnabled()) { + LOG.debug("{} completed for {}", useHsync ? "hsync" : "hflush", logFile); + } if (LOG.isTraceEnabled()) { LOG.trace("Sync slots=" + count + '/' + syncMaxSlot + ", flushed=" + StringUtils.humanSize(totalSynced)); @@ -1103,6 +1107,7 @@ public class WALProcedureStore extends ProcedureStoreBase { storeTracker.resetModified(); stream = newStream; + logFile = newLogFile; flushLogId = logId; totalSynced.set(0); long rollTs = System.currentTimeMillis(); @@ -1157,7 +1162,16 @@ public class WALProcedureStore extends ProcedureStoreBase { } catch (IOException e) { LOG.error("Unable to close the stream", e); } + if (LOG.isDebugEnabled()) { + try { + FileStatus fileStatus = fs.getFileStatus(logFile); + LOG.debug("After closing {}, length={}", logFile, fileStatus.getLen()); + } catch (IOException e) { + LOG.debug("Failed to get FileStatus for {} after closing.", logFile, e); + } + } stream = null; + logFile = null; } // ========================================================================== diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java index 881a7dfe94..52fab2e3cf 100644 --- a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java +++ b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java @@ -189,6 +189,7 @@ public class RSGroupBasedLoadBalancer implements RSGroupableBalancer { } } } + LOG.debug("round robin assign using " + assignments.size() + " servers, given "+servers.size()); return assignments; } @@ -233,6 +234,10 @@ public class RSGroupBasedLoadBalancer implements RSGroupableBalancer { } RSGroupInfo info = rsGroupInfoManager.getRSGroup(groupName); List candidateList = filterOfflineServers(info, servers); + if (candidateList.isEmpty() && region.getTable().isSystemTable()) { + LOG.debug("Should assign " + region + " to online servers though its group " + + groupName + " does't have any online server"); + } ServerName server = this.internalBalancer.randomAssignment(region, candidateList); if (server != null) { @@ -245,9 +250,11 @@ public class RSGroupBasedLoadBalancer implements RSGroupableBalancer { if(!assignments.containsKey(LoadBalancer.BOGUS_SERVER_NAME)) { assignments.put(LoadBalancer.BOGUS_SERVER_NAME, new ArrayList<>()); } + LOG.debug("adding " + region + " to " + LoadBalancer.BOGUS_SERVER_NAME); assignments.get(LoadBalancer.BOGUS_SERVER_NAME).add(region); } } + LOG.debug("retain assign using " + assignments.size() + " servers, given " + servers.size()); return assignments; } catch (IOException e) { throw new HBaseIOException("Failed to do online retain assignment", e); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java index d25d1ecdc3..ea4d827115 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java @@ -263,10 +263,19 @@ public class StochasticLoadBalancer extends BaseLoadBalancer { @Override protected synchronized boolean areSomeRegionReplicasColocated(Cluster c) { regionReplicaHostCostFunction.init(c); - if (regionReplicaHostCostFunction.cost() > 0) return true; + boolean ret = false; + double d = regionReplicaHostCostFunction.cost(); + if (d > 0) { + ret = true; + LOG.debug("replica on same host " + d); + } regionReplicaRackCostFunction.init(c); - if (regionReplicaRackCostFunction.cost() > 0) return true; - return false; + d = regionReplicaRackCostFunction.cost(); + if (d > 0) { + ret = true; + LOG.debug("replica on same rack " + d); + } + return ret; } @Override @@ -1607,10 +1616,14 @@ public class StochasticLoadBalancer extends BaseLoadBalancer { if (newHost != oldHost) { costsPerGroup[oldHost] = costPerGroup(cluster.primariesOfRegionsPerHost[oldHost]); costsPerGroup[newHost] = costPerGroup(cluster.primariesOfRegionsPerHost[newHost]); + LOG.debug("replica old cost " + costsPerGroup[oldHost] + " new cost " + + costsPerGroup[newHost]); } } else { costsPerGroup[oldServer] = costPerGroup(cluster.primariesOfRegionsPerServer[oldServer]); costsPerGroup[newServer] = costPerGroup(cluster.primariesOfRegionsPerServer[newServer]); + LOG.debug("replica cost on old server " + costsPerGroup[oldServer] + " cost on new server " + + costsPerGroup[newServer]); } } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableMemStoreLAB.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableMemStoreLAB.java index 0ce52b6fc7..ea61fd65af 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableMemStoreLAB.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableMemStoreLAB.java @@ -22,6 +22,8 @@ import java.util.concurrent.atomic.AtomicInteger; import org.apache.hadoop.hbase.Cell; import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * A MemStoreLAB implementation which wraps N MemStoreLABs. Its main duty is in proper managing the @@ -30,10 +32,11 @@ import org.apache.yetus.audience.InterfaceAudience; */ @InterfaceAudience.Private public class ImmutableMemStoreLAB implements MemStoreLAB { - + protected static final Logger LOG = LoggerFactory.getLogger(ImmutableMemStoreLAB.class); private final AtomicInteger openScannerCount = new AtomicInteger(); private volatile boolean closed = false; + private static final String MSG = "This is an Immutable MemStoreLAB."; private final List mslabs; public ImmutableMemStoreLAB(List mslabs) { @@ -42,6 +45,7 @@ public class ImmutableMemStoreLAB implements MemStoreLAB { @Override public Cell copyCellInto(Cell cell) { + LOG.error(MSG + cell.toString()); throw new IllegalStateException("This is an Immutable MemStoreLAB."); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java index 2c749d7952..c4643e9358 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java @@ -1608,9 +1608,11 @@ public class RSRpcServices implements HBaseRPCErrorHandler, requestCount.increment(); if (sn == null) { - LOG.info("Close " + encodedRegionName + " without moving"); + LOG.info("Close " + encodedRegionName + " without moving. Requested from " + + RpcServer.getRemoteAddress().orElse(null)); } else { - LOG.info("Close " + encodedRegionName + ", moving to " + sn); + LOG.info("Close " + encodedRegionName + ", moving to " + sn + ". Requested from " + + RpcServer.getRemoteAddress().orElse(null)); } boolean closed = regionServer.closeRegion(encodedRegionName, false, sn); CloseRegionResponse.Builder builder = CloseRegionResponse.newBuilder().setClosed(closed);