Index: src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java (revision 1221125) +++ src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java (working copy) @@ -296,6 +296,9 @@ // For measuring latency of syncs private static Metric syncTime = new Metric(); private static AtomicLong syncBatchSize = new AtomicLong(); + //For measuring slow HLog appends + private static AtomicLong slowHLogAppendCount = new AtomicLong(); + private static Metric slowHLogAppendTime = new Metric(); public static Metric getWriteTime() { return writeTime.get(); @@ -312,6 +315,14 @@ public static long getSyncBatchSize() { return syncBatchSize.getAndSet(0); } + + public static long getSlowAppendCount() { + return slowHLogAppendCount.get(); + } + + public static Metric getSlowAppendTime() { + return slowHLogAppendTime.get(); + } /** * Constructor. @@ -1407,6 +1418,8 @@ "%s took %d ms appending an edit to hlog; editcount=%d, len~=%s", Thread.currentThread().getName(), took, this.numEntries.get(), StringUtils.humanReadableInt(len))); + slowHLogAppendCount.incrementAndGet(); + slowHLogAppendTime.inc(took); } } catch (IOException e) { LOG.fatal("Could not append. Requesting close of hlog", e); Index: src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java (revision 1221125) +++ src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java (working copy) @@ -232,6 +232,12 @@ protected final PersistentMetricsTimeVaryingRate flushSize = new PersistentMetricsTimeVaryingRate("flushSize", registry); + + public final MetricsLongValue slowHLogAppendCount = + new MetricsLongValue("slowHLogAppendCount", registry); + + public final MetricsTimeVaryingRate slowHLogAppendTime = + new MetricsTimeVaryingRate("slowHLogAppendTime", registry); public RegionServerMetrics() { MetricsContext context = MetricsUtil.getContext("hbase"); @@ -323,6 +329,8 @@ addHLogMetric(HLog.getWriteTime(), this.fsWriteLatency); addHLogMetric(HLog.getWriteSize(), this.fsWriteSize); addHLogMetric(HLog.getSyncTime(), this.fsSyncLatency); + addHLogMetric(HLog.getSlowAppendTime(), this.slowHLogAppendTime); + this.slowHLogAppendCount.set(HLog.getSlowAppendCount()); // HFile metrics, sequential reads int ops = HFile.getReadOps(); if (ops != 0) this.fsReadLatency.inc(ops, HFile.getReadTimeMs()); @@ -345,6 +353,7 @@ this.compactionSize.pushMetric(this.metricsRecord); this.flushTime.pushMetric(this.metricsRecord); this.flushSize.pushMetric(this.metricsRecord); + this.slowHLogAppendCount.pushMetric(this.metricsRecord); } this.metricsRecord.update(); } @@ -367,6 +376,7 @@ this.fsWriteLatency.resetMinMax(); this.fsWriteSize.resetMinMax(); this.fsSyncLatency.resetMinMax(); + this.slowHLogAppendTime.resetMinMax(); } /** @@ -457,6 +467,8 @@ Long.valueOf(this.blockCacheHitCachingRatio.get())+"%"); sb = Strings.appendKeyValue(sb, this.hdfsBlocksLocalityIndex.getName(), Long.valueOf(this.hdfsBlocksLocalityIndex.get())); + sb = Strings.appendKeyValue(sb, "slowHLogAppendCount", + Long.valueOf(this.slowHLogAppendCount.get())); return sb.toString(); } }