Index: hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
===================================================================
--- hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java (revision 1548769)
+++ hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java (working copy)
@@ -117,6 +117,10 @@
String REGION_COUNT_DESC = "Number of regions";
String STORE_COUNT = "storeCount";
String STORE_COUNT_DESC = "Number of Stores";
+ String HLOGFILE_COUNT = "hlogFileCount";
+ String HLOGFILE_COUNT_DESC = "Number of HLog Files";
+ String HLOGFILE_SIZE = "hlogFileSize";
+ String HLOGFILE_SIZE_DESC = "Size of all HLog Files";
String STOREFILE_COUNT = "storeFileCount";
String STOREFILE_COUNT_DESC = "Number of Store Files";
String MEMSTORE_SIZE = "memStoreSize";
Index: hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
===================================================================
--- hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java (revision 1548769)
+++ hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java (working copy)
@@ -68,6 +68,16 @@
long getNumStores();
/**
+ * Get the number of HLog files of this region server.
+ */
+ public long getNumHLogFiles();
+
+ /**
+ * Get the size of HLog files of this region server.
+ */
+ public long getHLogFileSize();
+
+ /**
* Get the number of store files hosted on this region server.
*/
long getNumStoreFiles();
Index: hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
===================================================================
--- hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java (revision 1548769)
+++ hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java (working copy)
@@ -148,6 +148,8 @@
if (rsWrap != null) {
mrb.addGauge(REGION_COUNT, REGION_COUNT_DESC, rsWrap.getNumOnlineRegions())
.addGauge(STORE_COUNT, STORE_COUNT_DESC, rsWrap.getNumStores())
+ .addGauge(HLOGFILE_COUNT, HLOGFILE_COUNT_DESC, rsWrap.getNumHLogFiles())
+ .addGauge(HLOGFILE_SIZE, HLOGFILE_SIZE_DESC, rsWrap.getHLogFileSize())
.addGauge(STOREFILE_COUNT, STOREFILE_COUNT_DESC, rsWrap.getNumStoreFiles())
.addGauge(MEMSTORE_SIZE, MEMSTORE_SIZE_DESC, rsWrap.getMemstoreSize())
.addGauge(STOREFILE_SIZE, STOREFILE_SIZE_DESC, rsWrap.getStoreFileSize())
Index: hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
===================================================================
--- hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java (revision 1548769)
+++ hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java (working copy)
@@ -155,6 +155,8 @@
if (rsWrap != null) {
mrb.addGauge(Interns.info(REGION_COUNT, REGION_COUNT_DESC), rsWrap.getNumOnlineRegions())
.addGauge(Interns.info(STORE_COUNT, STORE_COUNT_DESC), rsWrap.getNumStores())
+ .addGauge(Interns.info(HLOGFILE_COUNT, HLOGFILE_COUNT_DESC), rsWrap.getNumHLogFiles())
+ .addGauge(Interns.info(HLOGFILE_SIZE, HLOGFILE_SIZE_DESC), rsWrap.getHLogFileSize())
.addGauge(Interns.info(STOREFILE_COUNT, STOREFILE_COUNT_DESC), rsWrap.getNumStoreFiles())
.addGauge(Interns.info(MEMSTORE_SIZE, MEMSTORE_SIZE_DESC), rsWrap.getMemstoreSize())
.addGauge(Interns.info(STOREFILE_SIZE, STOREFILE_SIZE_DESC), rsWrap.getStoreFileSize())
Index: hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon
===================================================================
--- hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon (revision 1548769)
+++ hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon (working copy)
@@ -40,6 +40,7 @@
Base Stats
Memory
Requests
+ hlogs
Storefiles
Queues
Block Cache
@@ -54,6 +55,9 @@
<& requestStats; mWrap = mWrap &>
+
+ <& hlogStats; mWrap = mWrap &>
+
<& storeStats; mWrap = mWrap &>
@@ -111,6 +115,24 @@
%def>
+<%def hlogStats>
+<%args>
+MetricsRegionServerWrapper mWrap;
+%args>
+
+
+
+ | Num. HLog Files |
+ Size. HLog Files (bytes) |
+
+
+
+ | <% mWrap.getNumHLogFiles() %> |
+ <% mWrap.getHLogFileSize() %> |
+
+
+%def>
+
<%def storeStats>
<%args>
MetricsRegionServerWrapper mWrap;
Index: hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
===================================================================
--- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java (revision 1548769)
+++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java (working copy)
@@ -50,6 +50,8 @@
private BlockCache blockCache;
private volatile long numStores = 0;
+ private volatile long numHLogFiles = 0;
+ private volatile long hlogFileSize = 0;
private volatile long numStoreFiles = 0;
private volatile long memstoreSize = 0;
private volatile long storeFileSize = 0;
@@ -263,8 +265,18 @@
public long getNumStores() {
return numStores;
}
+
+ @Override
+ public long getNumHLogFiles() {
+ return numHLogFiles;
+ }
@Override
+ public long getHLogFileSize() {
+ return hlogFileSize;
+ }
+
+ @Override
public long getNumStoreFiles() {
return numStoreFiles;
}
@@ -421,6 +433,19 @@
//Copy over computed values so that no thread sees half computed values.
numStores = tempNumStores;
+ long tempNumHLogFiles = regionServer.hlog.getNumLogFiles();
+ // meta logs
+ if (regionServer.hlogForMeta != null) {
+ tempNumHLogFiles += regionServer.hlogForMeta.getNumLogFiles();
+ }
+ numHLogFiles = tempNumHLogFiles;
+
+ long tempHlogFileSize = regionServer.hlog.getLogFileSize();
+ if (regionServer.hlogForMeta != null) {
+ tempHlogFileSize += regionServer.hlogForMeta.getLogFileSize();
+ }
+ hlogFileSize = tempHlogFileSize;
+
numStoreFiles = tempNumStoreFiles;
memstoreSize = tempMemstoreSize;
storeFileSize = tempStoreFileSize;
@@ -436,5 +461,4 @@
percentFileLocal = tempPercentFileLocal;
}
}
-
}
Index: hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
===================================================================
--- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java (revision 1548769)
+++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java (working copy)
@@ -203,6 +203,14 @@
// of the default Hdfs block size.
private final long logrollsize;
+ /** size of current log */
+ private long curLogSize = 0;
+
+ /**
+ * The total size of hlog
+ */
+ private AtomicLong totalLogSize = new AtomicLong(0);
+
// We synchronize on updateLock to prevent updates and to prevent a log roll
// during an update
// locked during appends
@@ -526,9 +534,14 @@
this.numEntries.set(0);
}
if (oldFile == null) LOG.info("New WAL " + FSUtils.getPath(newPath));
- else LOG.info("Rolled WAL " + FSUtils.getPath(oldFile) + " with entries=" + oldNumEntries +
- ", filesize=" + StringUtils.humanReadableInt(this.fs.getFileStatus(oldFile).getLen()) +
- "; new WAL " + FSUtils.getPath(newPath));
+ else {
+ long oldFileLen = this.fs.getFileStatus(oldFile).getLen();
+ this.totalLogSize.addAndGet(oldFileLen);
+ LOG.info("Rolled WAL " + FSUtils.getPath(oldFile) + " with entries="
+ + oldNumEntries + ", filesize="
+ + StringUtils.humanReadableInt(oldFileLen) + "; new WAL "
+ + FSUtils.getPath(newPath));
+ }
// Tell our listeners that a new log was created
if (!this.listeners.isEmpty()) {
@@ -538,7 +551,7 @@
}
// Can we delete any of the old log files?
- if (getNumLogFiles() > 0) {
+ if (getNumRolledLogFiles() > 0) {
cleanOldLogs();
regionsToFlush = getRegionsToForceFlush();
}
@@ -598,7 +611,9 @@
}
}
for (Long seq : sequenceNumbers) {
- archiveLogFile(this.outputfiles.remove(seq), seq);
+ Path p = this.outputfiles.remove(seq);
+ if (p != null) this.totalLogSize.addAndGet(-this.fs.getFileStatus(p).getLen());
+ archiveLogFile(p, seq);
}
}
@@ -626,7 +641,7 @@
// If too many log files, figure which regions we need to flush.
// Array is an array of encoded region names.
byte [][] regions = null;
- int logCount = getNumLogFiles();
+ int logCount = getNumRolledLogFiles();
if (logCount > this.maxLogs && logCount > 0) {
// This is an array of encoded region names.
synchronized (oldestSeqNumsLock) {
@@ -1096,7 +1111,8 @@
if (!this.logRollRunning) {
checkLowReplication();
try {
- if (tempWriter.getLength() > this.logrollsize) {
+ curLogSize = tempWriter.getLength();
+ if (curLogSize > this.logrollsize) {
requestLogRoll();
}
} catch (IOException x) {
@@ -1265,12 +1281,25 @@
return this.logSeqNum.incrementAndGet();
}
- /** @return the number of log files in use */
- int getNumLogFiles() {
+ /** @return the number of rolled log files */
+ public int getNumRolledLogFiles() {
return outputfiles.size();
}
+ /** @return the number of log files in use */
@Override
+ public int getNumLogFiles() {
+ // +1 for current use log
+ return getNumRolledLogFiles() + 1;
+ }
+
+ /** @return the size of log files in use */
+ @Override
+ public long getLogFileSize() {
+ return totalLogSize.get() + curLogSize;
+ }
+
+ @Override
public Long startCacheFlush(final byte[] encodedRegionName) {
Long oldRegionSeqNum = null;
if (!closeBarrier.beginOp()) {
Index: hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
===================================================================
--- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java (revision 1548769)
+++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java (working copy)
@@ -218,6 +218,16 @@
*/
long getSequenceNumber();
+ /**
+ * @return the number of HLog files
+ */
+ int getNumLogFiles();
+
+ /**
+ * @return the size of HLog files
+ */
+ long getLogFileSize();
+
// TODO: Log rolling should not be in this interface.
/**
* Roll the log writer. That is, start writing log messages to a new file.
Index: hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
===================================================================
--- hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java (revision 1548769)
+++ hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java (working copy)
@@ -1523,7 +1523,7 @@
byte[] value = Bytes.toBytes(v.toString());
HRegionServer regionServer = startAndWriteData("TestLogRolling", value);
LOG.info("after writing there are "
- + HLogUtilsForTests.getNumLogFiles(regionServer.getWAL()) + " log files");
+ + HLogUtilsForTests.getNumRolledLogFiles(regionServer.getWAL()) + " log files");
// flush all regions
@@ -1533,7 +1533,7 @@
r.flushcache();
}
admin.rollHLogWriter(regionServer.getServerName().getServerName());
- int count = HLogUtilsForTests.getNumLogFiles(regionServer.getWAL());
+ int count = HLogUtilsForTests.getNumRolledLogFiles(regionServer.getWAL());
LOG.info("after flushing all regions and rolling logs there are " +
count + " log files");
assertTrue(("actual count: " + count), count <= 2);
Index: hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
===================================================================
--- hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java (revision 1548769)
+++ hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java (working copy)
@@ -201,4 +201,14 @@
//IGNORED.
}
+ @Override
+ public long getNumHLogFiles() {
+ return 10;
+ }
+
+ @Override
+ public long getHLogFileSize() {
+ return 1024000;
+ }
+
}
\ No newline at end of file
Index: hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java
===================================================================
--- hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java (revision 1548769)
+++ hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java (working copy)
@@ -59,6 +59,8 @@
HELPER.assertGauge("regionServerStartTime", 100, serverSource);
HELPER.assertGauge("regionCount", 101, serverSource);
HELPER.assertGauge("storeCount", 2, serverSource);
+ HELPER.assertGauge("hlogFileCount", 10, serverSource);
+ HELPER.assertGauge("hlogFileSize", 1024000, serverSource);
HELPER.assertGauge("storeFileCount", 300, serverSource);
HELPER.assertGauge("memstoreSize", 1025, serverSource);
HELPER.assertGauge("storeFileSize", 1900, serverSource);
Index: hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/HLogUtilsForTests.java
===================================================================
--- hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/HLogUtilsForTests.java (revision 1548769)
+++ hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/HLogUtilsForTests.java (working copy)
@@ -33,8 +33,8 @@
* @param log
* @return
*/
- public static int getNumLogFiles(HLog log) {
- return ((FSHLog) log).getNumLogFiles();
+ public static int getNumRolledLogFiles(HLog log) {
+ return ((FSHLog) log).getNumRolledLogFiles();
}
public static int getNumEntries(HLog log) {
Index: hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
===================================================================
--- hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java (revision 1548769)
+++ hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java (working copy)
@@ -675,12 +675,12 @@
// Before HBASE-3198 it used to delete it
addEdits(log, hri, tableName, 1);
log.rollWriter();
- assertEquals(1, ((FSHLog) log).getNumLogFiles());
+ assertEquals(1, ((FSHLog) log).getNumRolledLogFiles());
// See if there's anything wrong with more than 1 edit
addEdits(log, hri, tableName, 2);
log.rollWriter();
- assertEquals(2, ((FSHLog) log).getNumLogFiles());
+ assertEquals(2, ((FSHLog) log).getNumRolledLogFiles());
// Now mix edits from 2 regions, still no flushing
addEdits(log, hri, tableName, 1);
@@ -688,7 +688,7 @@
addEdits(log, hri, tableName, 1);
addEdits(log, hri2, tableName2, 1);
log.rollWriter();
- assertEquals(3, ((FSHLog) log).getNumLogFiles());
+ assertEquals(3, ((FSHLog) log).getNumRolledLogFiles());
// Flush the first region, we expect to see the first two files getting
// archived. We need to append something or writer won't be rolled.
@@ -696,7 +696,7 @@
log.startCacheFlush(hri.getEncodedNameAsBytes());
log.completeCacheFlush(hri.getEncodedNameAsBytes());
log.rollWriter();
- assertEquals(2, ((FSHLog) log).getNumLogFiles());
+ assertEquals(2, ((FSHLog) log).getNumRolledLogFiles());
// Flush the second region, which removes all the remaining output files
// since the oldest was completely flushed and the two others only contain
@@ -705,7 +705,7 @@
log.startCacheFlush(hri2.getEncodedNameAsBytes());
log.completeCacheFlush(hri2.getEncodedNameAsBytes());
log.rollWriter();
- assertEquals(0, ((FSHLog) log).getNumLogFiles());
+ assertEquals(0, ((FSHLog) log).getNumRolledLogFiles());
} finally {
if (log != null) log.closeAndDelete();
}
Index: hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java
===================================================================
--- hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java (revision 1548769)
+++ hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java (working copy)
@@ -221,7 +221,7 @@
public void testLogRolling() throws Exception {
this.tableName = getName();
startAndWriteData();
- LOG.info("after writing there are " + ((FSHLog) log).getNumLogFiles() + " log files");
+ LOG.info("after writing there are " + ((FSHLog) log).getNumRolledLogFiles() + " log files");
// flush all regions
@@ -234,9 +234,9 @@
// Now roll the log
log.rollWriter();
- int count = ((FSHLog) log).getNumLogFiles();
+ int count = ((FSHLog) log).getNumRolledLogFiles();
LOG.info("after flushing all regions and rolling logs there are " +
- ((FSHLog) log).getNumLogFiles() + " log files");
+ ((FSHLog) log).getNumRolledLogFiles() + " log files");
assertTrue(("actual count: " + count), count <= 2);
}
@@ -606,12 +606,12 @@
admin.flush(table2.getTableName());
}
doPut(table2, 3); // don't flush yet, or compaction might trigger before we roll WAL
- assertEquals("Should have no WAL after initial writes", 0, fshLog.getNumLogFiles());
+ assertEquals("Should have no WAL after initial writes", 0, fshLog.getNumRolledLogFiles());
assertEquals(2, s.getStorefilesCount());
// Roll the log and compact table2, to have compaction record in the 2nd WAL.
fshLog.rollWriter();
- assertEquals("Should have WAL; one table is not flushed", 1, fshLog.getNumLogFiles());
+ assertEquals("Should have WAL; one table is not flushed", 1, fshLog.getNumRolledLogFiles());
admin.flush(table2.getTableName());
region.compactStores();
// Wait for compaction in case if flush triggered it before us.
@@ -624,13 +624,13 @@
// Write some value to the table so the WAL cannot be deleted until table is flushed.
doPut(table, 0); // Now 2nd WAL will have compaction record for table2 and put for table.
fshLog.rollWriter(); // 1st WAL deleted, 2nd not deleted yet.
- assertEquals("Should have WAL; one table is not flushed", 1, fshLog.getNumLogFiles());
+ assertEquals("Should have WAL; one table is not flushed", 1, fshLog.getNumRolledLogFiles());
// Flush table to make latest WAL obsolete; write another record, and roll again.
admin.flush(table.getTableName());
doPut(table, 1);
fshLog.rollWriter(); // Now 2nd WAL is deleted and 3rd is added.
- assertEquals("Should have 1 WALs at the end", 1, fshLog.getNumLogFiles());
+ assertEquals("Should have 1 WALs at the end", 1, fshLog.getNumRolledLogFiles());
table.close();
table2.close();