Index: hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java =================================================================== --- hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java (revision 1548769) +++ hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java (working copy) @@ -117,6 +117,10 @@ String REGION_COUNT_DESC = "Number of regions"; String STORE_COUNT = "storeCount"; String STORE_COUNT_DESC = "Number of Stores"; + String HLOGFILE_COUNT = "hlogFileCount"; + String HLOGFILE_COUNT_DESC = "Number of HLog Files"; + String HLOGFILE_SIZE = "hlogFileSize"; + String HLOGFILE_SIZE_DESC = "Size of all HLog Files"; String STOREFILE_COUNT = "storeFileCount"; String STOREFILE_COUNT_DESC = "Number of Store Files"; String MEMSTORE_SIZE = "memStoreSize"; Index: hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java =================================================================== --- hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java (revision 1548769) +++ hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java (working copy) @@ -68,6 +68,16 @@ long getNumStores(); /** + * Get the number of HLog files of this region server. + */ + public long getNumHLogFiles(); + + /** + * Get the size of HLog files of this region server. + */ + public long getHLogFileSize(); + + /** * Get the number of store files hosted on this region server. */ long getNumStoreFiles(); Index: hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java =================================================================== --- hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java (revision 1548769) +++ hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java (working copy) @@ -148,6 +148,8 @@ if (rsWrap != null) { mrb.addGauge(REGION_COUNT, REGION_COUNT_DESC, rsWrap.getNumOnlineRegions()) .addGauge(STORE_COUNT, STORE_COUNT_DESC, rsWrap.getNumStores()) + .addGauge(HLOGFILE_COUNT, HLOGFILE_COUNT_DESC, rsWrap.getNumHLogFiles()) + .addGauge(HLOGFILE_SIZE, HLOGFILE_SIZE_DESC, rsWrap.getHLogFileSize()) .addGauge(STOREFILE_COUNT, STOREFILE_COUNT_DESC, rsWrap.getNumStoreFiles()) .addGauge(MEMSTORE_SIZE, MEMSTORE_SIZE_DESC, rsWrap.getMemstoreSize()) .addGauge(STOREFILE_SIZE, STOREFILE_SIZE_DESC, rsWrap.getStoreFileSize()) Index: hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java =================================================================== --- hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java (revision 1548769) +++ hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java (working copy) @@ -155,6 +155,8 @@ if (rsWrap != null) { mrb.addGauge(Interns.info(REGION_COUNT, REGION_COUNT_DESC), rsWrap.getNumOnlineRegions()) .addGauge(Interns.info(STORE_COUNT, STORE_COUNT_DESC), rsWrap.getNumStores()) + .addGauge(Interns.info(HLOGFILE_COUNT, HLOGFILE_COUNT_DESC), rsWrap.getNumHLogFiles()) + .addGauge(Interns.info(HLOGFILE_SIZE, HLOGFILE_SIZE_DESC), rsWrap.getHLogFileSize()) .addGauge(Interns.info(STOREFILE_COUNT, STOREFILE_COUNT_DESC), rsWrap.getNumStoreFiles()) .addGauge(Interns.info(MEMSTORE_SIZE, MEMSTORE_SIZE_DESC), rsWrap.getMemstoreSize()) .addGauge(Interns.info(STOREFILE_SIZE, STOREFILE_SIZE_DESC), rsWrap.getStoreFileSize()) Index: hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon =================================================================== --- hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon (revision 1548769) +++ hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon (working copy) @@ -40,6 +40,7 @@
  • Base Stats
  • Memory
  • Requests
  • +
  • hlogs
  • Storefiles
  • Queues
  • Block Cache
  • @@ -54,6 +55,9 @@
    <& requestStats; mWrap = mWrap &>
    +
    + <& hlogStats; mWrap = mWrap &> +
    <& storeStats; mWrap = mWrap &>
    @@ -111,6 +115,24 @@ +<%def hlogStats> +<%args> +MetricsRegionServerWrapper mWrap; + + + + + + + + + + + + +
    Num. HLog FilesSize. HLog Files (bytes)
    <% mWrap.getNumHLogFiles() %><% mWrap.getHLogFileSize() %>
    + + <%def storeStats> <%args> MetricsRegionServerWrapper mWrap; Index: hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java (revision 1548769) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java (working copy) @@ -50,6 +50,8 @@ private BlockCache blockCache; private volatile long numStores = 0; + private volatile long numHLogFiles = 0; + private volatile long hlogFileSize = 0; private volatile long numStoreFiles = 0; private volatile long memstoreSize = 0; private volatile long storeFileSize = 0; @@ -263,8 +265,18 @@ public long getNumStores() { return numStores; } + + @Override + public long getNumHLogFiles() { + return numHLogFiles; + } @Override + public long getHLogFileSize() { + return hlogFileSize; + } + + @Override public long getNumStoreFiles() { return numStoreFiles; } @@ -421,6 +433,19 @@ //Copy over computed values so that no thread sees half computed values. numStores = tempNumStores; + long tempNumHLogFiles = regionServer.hlog.getNumLogFiles(); + // meta logs + if (regionServer.hlogForMeta != null) { + tempNumHLogFiles += regionServer.hlogForMeta.getNumLogFiles(); + } + numHLogFiles = tempNumHLogFiles; + + long tempHlogFileSize = regionServer.hlog.getLogFileSize(); + if (regionServer.hlogForMeta != null) { + tempHlogFileSize += regionServer.hlogForMeta.getLogFileSize(); + } + hlogFileSize = tempHlogFileSize; + numStoreFiles = tempNumStoreFiles; memstoreSize = tempMemstoreSize; storeFileSize = tempStoreFileSize; @@ -436,5 +461,4 @@ percentFileLocal = tempPercentFileLocal; } } - } Index: hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java (revision 1548769) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java (working copy) @@ -203,6 +203,14 @@ // of the default Hdfs block size. private final long logrollsize; + /** size of current log */ + private long curLogSize = 0; + + /** + * The total size of hlog + */ + private AtomicLong totalLogSize = new AtomicLong(0); + // We synchronize on updateLock to prevent updates and to prevent a log roll // during an update // locked during appends @@ -526,9 +534,14 @@ this.numEntries.set(0); } if (oldFile == null) LOG.info("New WAL " + FSUtils.getPath(newPath)); - else LOG.info("Rolled WAL " + FSUtils.getPath(oldFile) + " with entries=" + oldNumEntries + - ", filesize=" + StringUtils.humanReadableInt(this.fs.getFileStatus(oldFile).getLen()) + - "; new WAL " + FSUtils.getPath(newPath)); + else { + long oldFileLen = this.fs.getFileStatus(oldFile).getLen(); + this.totalLogSize.addAndGet(oldFileLen); + LOG.info("Rolled WAL " + FSUtils.getPath(oldFile) + " with entries=" + + oldNumEntries + ", filesize=" + + StringUtils.humanReadableInt(oldFileLen) + "; new WAL " + + FSUtils.getPath(newPath)); + } // Tell our listeners that a new log was created if (!this.listeners.isEmpty()) { @@ -538,7 +551,7 @@ } // Can we delete any of the old log files? - if (getNumLogFiles() > 0) { + if (getNumRolledLogFiles() > 0) { cleanOldLogs(); regionsToFlush = getRegionsToForceFlush(); } @@ -598,7 +611,9 @@ } } for (Long seq : sequenceNumbers) { - archiveLogFile(this.outputfiles.remove(seq), seq); + Path p = this.outputfiles.remove(seq); + if (p != null) this.totalLogSize.addAndGet(-this.fs.getFileStatus(p).getLen()); + archiveLogFile(p, seq); } } @@ -626,7 +641,7 @@ // If too many log files, figure which regions we need to flush. // Array is an array of encoded region names. byte [][] regions = null; - int logCount = getNumLogFiles(); + int logCount = getNumRolledLogFiles(); if (logCount > this.maxLogs && logCount > 0) { // This is an array of encoded region names. synchronized (oldestSeqNumsLock) { @@ -1096,7 +1111,8 @@ if (!this.logRollRunning) { checkLowReplication(); try { - if (tempWriter.getLength() > this.logrollsize) { + curLogSize = tempWriter.getLength(); + if (curLogSize > this.logrollsize) { requestLogRoll(); } } catch (IOException x) { @@ -1265,12 +1281,25 @@ return this.logSeqNum.incrementAndGet(); } - /** @return the number of log files in use */ - int getNumLogFiles() { + /** @return the number of rolled log files */ + public int getNumRolledLogFiles() { return outputfiles.size(); } + /** @return the number of log files in use */ @Override + public int getNumLogFiles() { + // +1 for current use log + return getNumRolledLogFiles() + 1; + } + + /** @return the size of log files in use */ + @Override + public long getLogFileSize() { + return totalLogSize.get() + curLogSize; + } + + @Override public Long startCacheFlush(final byte[] encodedRegionName) { Long oldRegionSeqNum = null; if (!closeBarrier.beginOp()) { Index: hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java (revision 1548769) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java (working copy) @@ -218,6 +218,16 @@ */ long getSequenceNumber(); + /** + * @return the number of HLog files + */ + int getNumLogFiles(); + + /** + * @return the size of HLog files + */ + long getLogFileSize(); + // TODO: Log rolling should not be in this interface. /** * Roll the log writer. That is, start writing log messages to a new file. Index: hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java =================================================================== --- hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java (revision 1548769) +++ hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java (working copy) @@ -1523,7 +1523,7 @@ byte[] value = Bytes.toBytes(v.toString()); HRegionServer regionServer = startAndWriteData("TestLogRolling", value); LOG.info("after writing there are " - + HLogUtilsForTests.getNumLogFiles(regionServer.getWAL()) + " log files"); + + HLogUtilsForTests.getNumRolledLogFiles(regionServer.getWAL()) + " log files"); // flush all regions @@ -1533,7 +1533,7 @@ r.flushcache(); } admin.rollHLogWriter(regionServer.getServerName().getServerName()); - int count = HLogUtilsForTests.getNumLogFiles(regionServer.getWAL()); + int count = HLogUtilsForTests.getNumRolledLogFiles(regionServer.getWAL()); LOG.info("after flushing all regions and rolling logs there are " + count + " log files"); assertTrue(("actual count: " + count), count <= 2); Index: hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java =================================================================== --- hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java (revision 1548769) +++ hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java (working copy) @@ -201,4 +201,14 @@ //IGNORED. } + @Override + public long getNumHLogFiles() { + return 10; + } + + @Override + public long getHLogFileSize() { + return 1024000; + } + } \ No newline at end of file Index: hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java =================================================================== --- hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java (revision 1548769) +++ hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java (working copy) @@ -59,6 +59,8 @@ HELPER.assertGauge("regionServerStartTime", 100, serverSource); HELPER.assertGauge("regionCount", 101, serverSource); HELPER.assertGauge("storeCount", 2, serverSource); + HELPER.assertGauge("hlogFileCount", 10, serverSource); + HELPER.assertGauge("hlogFileSize", 1024000, serverSource); HELPER.assertGauge("storeFileCount", 300, serverSource); HELPER.assertGauge("memstoreSize", 1025, serverSource); HELPER.assertGauge("storeFileSize", 1900, serverSource); Index: hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/HLogUtilsForTests.java =================================================================== --- hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/HLogUtilsForTests.java (revision 1548769) +++ hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/HLogUtilsForTests.java (working copy) @@ -33,8 +33,8 @@ * @param log * @return */ - public static int getNumLogFiles(HLog log) { - return ((FSHLog) log).getNumLogFiles(); + public static int getNumRolledLogFiles(HLog log) { + return ((FSHLog) log).getNumRolledLogFiles(); } public static int getNumEntries(HLog log) { Index: hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java =================================================================== --- hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java (revision 1548769) +++ hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java (working copy) @@ -675,12 +675,12 @@ // Before HBASE-3198 it used to delete it addEdits(log, hri, tableName, 1); log.rollWriter(); - assertEquals(1, ((FSHLog) log).getNumLogFiles()); + assertEquals(1, ((FSHLog) log).getNumRolledLogFiles()); // See if there's anything wrong with more than 1 edit addEdits(log, hri, tableName, 2); log.rollWriter(); - assertEquals(2, ((FSHLog) log).getNumLogFiles()); + assertEquals(2, ((FSHLog) log).getNumRolledLogFiles()); // Now mix edits from 2 regions, still no flushing addEdits(log, hri, tableName, 1); @@ -688,7 +688,7 @@ addEdits(log, hri, tableName, 1); addEdits(log, hri2, tableName2, 1); log.rollWriter(); - assertEquals(3, ((FSHLog) log).getNumLogFiles()); + assertEquals(3, ((FSHLog) log).getNumRolledLogFiles()); // Flush the first region, we expect to see the first two files getting // archived. We need to append something or writer won't be rolled. @@ -696,7 +696,7 @@ log.startCacheFlush(hri.getEncodedNameAsBytes()); log.completeCacheFlush(hri.getEncodedNameAsBytes()); log.rollWriter(); - assertEquals(2, ((FSHLog) log).getNumLogFiles()); + assertEquals(2, ((FSHLog) log).getNumRolledLogFiles()); // Flush the second region, which removes all the remaining output files // since the oldest was completely flushed and the two others only contain @@ -705,7 +705,7 @@ log.startCacheFlush(hri2.getEncodedNameAsBytes()); log.completeCacheFlush(hri2.getEncodedNameAsBytes()); log.rollWriter(); - assertEquals(0, ((FSHLog) log).getNumLogFiles()); + assertEquals(0, ((FSHLog) log).getNumRolledLogFiles()); } finally { if (log != null) log.closeAndDelete(); } Index: hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java =================================================================== --- hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java (revision 1548769) +++ hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java (working copy) @@ -221,7 +221,7 @@ public void testLogRolling() throws Exception { this.tableName = getName(); startAndWriteData(); - LOG.info("after writing there are " + ((FSHLog) log).getNumLogFiles() + " log files"); + LOG.info("after writing there are " + ((FSHLog) log).getNumRolledLogFiles() + " log files"); // flush all regions @@ -234,9 +234,9 @@ // Now roll the log log.rollWriter(); - int count = ((FSHLog) log).getNumLogFiles(); + int count = ((FSHLog) log).getNumRolledLogFiles(); LOG.info("after flushing all regions and rolling logs there are " + - ((FSHLog) log).getNumLogFiles() + " log files"); + ((FSHLog) log).getNumRolledLogFiles() + " log files"); assertTrue(("actual count: " + count), count <= 2); } @@ -606,12 +606,12 @@ admin.flush(table2.getTableName()); } doPut(table2, 3); // don't flush yet, or compaction might trigger before we roll WAL - assertEquals("Should have no WAL after initial writes", 0, fshLog.getNumLogFiles()); + assertEquals("Should have no WAL after initial writes", 0, fshLog.getNumRolledLogFiles()); assertEquals(2, s.getStorefilesCount()); // Roll the log and compact table2, to have compaction record in the 2nd WAL. fshLog.rollWriter(); - assertEquals("Should have WAL; one table is not flushed", 1, fshLog.getNumLogFiles()); + assertEquals("Should have WAL; one table is not flushed", 1, fshLog.getNumRolledLogFiles()); admin.flush(table2.getTableName()); region.compactStores(); // Wait for compaction in case if flush triggered it before us. @@ -624,13 +624,13 @@ // Write some value to the table so the WAL cannot be deleted until table is flushed. doPut(table, 0); // Now 2nd WAL will have compaction record for table2 and put for table. fshLog.rollWriter(); // 1st WAL deleted, 2nd not deleted yet. - assertEquals("Should have WAL; one table is not flushed", 1, fshLog.getNumLogFiles()); + assertEquals("Should have WAL; one table is not flushed", 1, fshLog.getNumRolledLogFiles()); // Flush table to make latest WAL obsolete; write another record, and roll again. admin.flush(table.getTableName()); doPut(table, 1); fshLog.rollWriter(); // Now 2nd WAL is deleted and 3rd is added. - assertEquals("Should have 1 WALs at the end", 1, fshLog.getNumLogFiles()); + assertEquals("Should have 1 WALs at the end", 1, fshLog.getNumRolledLogFiles()); table.close(); table2.close();