Index: src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java (revision 1332868) +++ src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java (working copy) @@ -210,6 +210,18 @@ new MetricsIntValue("memstoreSizeMB", registry); /** + * Number of put with WAL disabled in this regionserver in MB + */ + public final MetricsLongValue numPutsWithoutWAL = + new MetricsLongValue("numPutsWithoutWAL", registry); + + /** + * Possible data loss sizes (due to put with WAL disabled) in this regionserver in MB + */ + public final MetricsIntValue mbInMemoryWithoutWAL = + new MetricsIntValue("mbInMemoryWithoutWAL", registry); + + /** * Size of the compaction queue. */ public final MetricsIntValue compactionQueueSize = @@ -366,6 +378,8 @@ this.totalStaticIndexSizeKB.pushMetric(this.metricsRecord); this.totalStaticBloomSizeKB.pushMetric(this.metricsRecord); this.memstoreSizeMB.pushMetric(this.metricsRecord); + this.mbInMemoryWithoutWAL.pushMetric(this.metricsRecord); + this.numPutsWithoutWAL.pushMetric(this.metricsRecord); this.readRequestsCount.pushMetric(this.metricsRecord); this.writeRequestsCount.pushMetric(this.metricsRecord); this.regions.pushMetric(this.metricsRecord); @@ -535,6 +549,10 @@ Integer.valueOf(this.totalStaticBloomSizeKB.get())); sb = Strings.appendKeyValue(sb, this.memstoreSizeMB.getName(), Integer.valueOf(this.memstoreSizeMB.get())); + sb = Strings.appendKeyValue(sb, "mbInMemoryWithoutWAL", + Integer.valueOf(this.mbInMemoryWithoutWAL.get())); + sb = Strings.appendKeyValue(sb, "numberOfPutsWithoutWAL", + Long.valueOf(this.numPutsWithoutWAL.get())); sb = Strings.appendKeyValue(sb, "readRequestsCount", Long.valueOf(this.readRequestsCount.get())); sb = Strings.appendKeyValue(sb, "writeRequestsCount", Index: src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java (revision 1332868) +++ src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java (working copy) @@ -220,6 +220,10 @@ final AtomicLong memstoreSize = new AtomicLong(0); + // Debug possible data loss due to WAL off + final AtomicLong numPutsWithoutWAL = new AtomicLong(0); + final AtomicLong dataInMemoryWithoutWAL = new AtomicLong(0); + final Counter readRequestsCount = new Counter(); final Counter writeRequestsCount = new Counter(); @@ -1316,6 +1320,10 @@ status.setStatus("Running coprocessor pre-flush hooks"); coprocessorHost.preFlush(); } + if (numPutsWithoutWAL.get() > 0) { + numPutsWithoutWAL.set(0); + dataInMemoryWithoutWAL.set(0); + } synchronized (writestate) { if (!writestate.flushing && writestate.writesEnabled) { this.writestate.flushing = true; @@ -2169,7 +2177,10 @@ batchOp.retCodeDetails[i] = OperationStatus.SUCCESS; Put p = batchOp.operations[i].getFirst(); - if (!p.getWriteToWAL()) continue; + if (!p.getWriteToWAL()) { + recordPutWithoutWal(p.getFamilyMap()); + continue; + } // Add WAL edits by CP WALEdit fromCP = batchOp.walEditsFromCoprocessors[i]; if (fromCP != null) { @@ -2502,6 +2513,8 @@ addFamilyMapToWALEdit(familyMap, walEdit); this.log.append(regionInfo, this.htableDescriptor.getName(), walEdit, clusterId, now, this.htableDescriptor); + } else { + recordPutWithoutWal(familyMap); } long addedSize = applyFamilyMapToMemstore(familyMap, null); @@ -4823,14 +4836,14 @@ public static final long FIXED_OVERHEAD = ClassSize.align( ClassSize.OBJECT + ClassSize.ARRAY + - 34 * ClassSize.REFERENCE + Bytes.SIZEOF_INT + + 36 * ClassSize.REFERENCE + Bytes.SIZEOF_INT + (6 * Bytes.SIZEOF_LONG) + Bytes.SIZEOF_BOOLEAN); public static final long DEEP_OVERHEAD = FIXED_OVERHEAD + ClassSize.OBJECT + // closeLock (2 * ClassSize.ATOMIC_BOOLEAN) + // closed, closing - ClassSize.ATOMIC_LONG + // memStoreSize + (3 * ClassSize.ATOMIC_LONG) + // memStoreSize, numPutsWithoutWAL, dataInMemoryWithoutWAL ClassSize.ATOMIC_INTEGER + // lockIdGenerator (3 * ClassSize.CONCURRENT_HASHMAP) + // lockedRows, lockIds, scannerReadPoints WriteState.HEAP_SIZE + // writestate @@ -5179,6 +5192,26 @@ } /** + * Update counters for numer of puts without wal and the size of possible data loss. + * These information are exposed by the region server metrics. + */ + private void recordPutWithoutWal(final Map> familyMap) { + if (numPutsWithoutWAL.getAndIncrement() == 0) { + LOG.info("writing data to region " + this + + " with WAL disabled. Data may be lost in the event of a crash."); + } + + long putSize = 0; + for (List edits : familyMap.values()) { + for (KeyValue kv : edits) { + putSize += kv.getKeyLength() + kv.getValueLength(); + } + } + + dataInMemoryWithoutWAL.addAndGet(putSize); + } + + /** * A mocked list implementaion - discards all updates. */ private static final List MOCKED_LIST = new AbstractList() { Index: src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (revision 1332868) +++ src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (working copy) @@ -1147,6 +1147,8 @@ new HDFSBlocksDistribution(); long totalStaticIndexSize = 0; long totalStaticBloomSize = 0; + long numPutsWithoutWAL = 0; + long dataInMemoryWithoutWAL = 0; // Note that this is a map of Doubles instead of Longs. This is because we // do effective integer division, which would perhaps truncate more than it @@ -1159,6 +1161,8 @@ for (Map.Entry e : this.onlineRegions.entrySet()) { HRegion r = e.getValue(); memstoreSize += r.memstoreSize.get(); + numPutsWithoutWAL += r.numPutsWithoutWAL.get(); + dataInMemoryWithoutWAL += r.dataInMemoryWithoutWAL.get(); readRequestsCount += r.readRequestsCount.get(); writeRequestsCount += r.writeRequestsCount.get(); synchronized (r.stores) { @@ -1222,6 +1226,8 @@ this.metrics.stores.set(stores); this.metrics.storefiles.set(storefiles); this.metrics.memstoreSizeMB.set((int) (memstoreSize / (1024 * 1024))); + this.metrics.mbInMemoryWithoutWAL.set((int) (dataInMemoryWithoutWAL / (1024 * 1024))); + this.metrics.numPutsWithoutWAL.set(numPutsWithoutWAL); this.metrics.storefileIndexSizeMB.set( (int) (storefileIndexSize / (1024 * 1024))); this.metrics.rootIndexSizeKB.set(