diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
index 768da76..409fc89 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
@@ -67,6 +67,7 @@ import org.apache.hadoop.hbase.exceptions.DeserializationException;
import org.apache.hadoop.hbase.filter.ByteArrayComparable;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.io.TimeRange;
+import org.apache.hadoop.hbase.io.hfile.CellStatsTracker;
import org.apache.hadoop.hbase.protobuf.generated.AccessControlProtos;
import org.apache.hadoop.hbase.protobuf.generated.AccessControlProtos.AccessControlService;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService;
@@ -86,6 +87,7 @@ import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ServerInfo;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.SplitRegionRequest;
import org.apache.hadoop.hbase.protobuf.generated.AuthenticationProtos;
import org.apache.hadoop.hbase.protobuf.generated.CellProtos;
+import org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.BulkLoadHFileRequest;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.BulkLoadHFileResponse;
@@ -2964,4 +2966,23 @@ public final class ProtobufUtil {
.setScope(toProtoQuotaScope(scope))
.build();
}
+
+ public static CellStatsTracker ToCellStatsTracker(CellTrackerProtos.CellStatsTracker proto)
+ throws IOException {
+ return new CellStatsTracker(proto.getColCount(), proto.getRowCount(), proto.getByteCount(),
+ proto.getMaxColSize(), proto.getMaxRowSize(), proto.getPreColSizeVar(),
+ proto.getPreRowSizeVar());
+ }
+
+ public static CellTrackerProtos.CellStatsTracker toCellStatsTracker(CellStatsTracker tracker)
+ throws IOException {
+ CellTrackerProtos.CellStatsTracker.Builder builder =
+ CellTrackerProtos.CellStatsTracker.newBuilder();
+ builder.setByteCount(tracker.getByteCount()).setColCount(tracker.getColCount())
+ .setRowCount(tracker.getRowCount()).setMaxColSize(tracker.getMaxColSize())
+ .setMaxRowSize(tracker.getMaxRowSize()).setPreColSizeVar(tracker.getPreColSizeVariance())
+ .setPreRowSizeVar(tracker.getPreRowSizeVariance());
+ return builder.build();
+ }
+
}
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/hfile/CellStatsTracker.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/hfile/CellStatsTracker.java
new file mode 100644
index 0000000..517f249
--- /dev/null
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/hfile/CellStatsTracker.java
@@ -0,0 +1,145 @@
+package org.apache.hadoop.hbase.io.hfile;
+
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+
+public class CellStatsTracker {
+ private Cell row = null;
+ private Cell col = null;
+
+ private long colCount;
+ private long rowCount;
+ private long byteCount;
+
+ private long curColSize = 0;
+ private long curRowSize = 0;
+
+ private long maxColSize;
+ private long maxRowSize;
+
+ private double preColSizeVariance;
+ private double preRowSizeVariance;
+
+ private int FIXED_CELL_LENGTH = 16; // is this right?
+
+ public CellStatsTracker() {}
+
+ // used to deserialize from PB
+ public CellStatsTracker(long colCount, long rowCount, long byteCount, long maxColSize, long maxRowSize, double preColSizeVariance, double preRowSizeVariance) {
+ this.colCount = colCount;
+ this.rowCount = rowCount;
+ this.byteCount = byteCount;
+ this.maxColSize = maxColSize;
+ this.maxRowSize = maxRowSize;
+ this.preColSizeVariance = preColSizeVariance;
+ this.preRowSizeVariance = preRowSizeVariance;
+ }
+
+ public void combine(CellStatsTracker other) {
+ this.colCount += other.colCount;
+ this.rowCount += other.rowCount;
+ this.byteCount += other.byteCount;
+ this.maxColSize = Math.max(maxColSize, other.maxColSize);
+ this.maxRowSize = Math.max(maxRowSize, other.maxRowSize);
+ this.preColSizeVariance = combinePreVariance(this.getColSizeVariance(), this.getColCount(), other.getColSizeVariance(), other.getColCount());
+ this.preRowSizeVariance = combinePreVariance(this.getRowSizeVariance(), this.getRowCount(), other.getRowSizeVariance(), other.getRowCount());
+ }
+
+ private double combinePreVariance(double v1, long sample1, double v2, long sample2) {
+ if (sample1 == 0) return v2;
+ if (sample2 == 0) return v1;
+ return (v1/sample1+v2/sample2)*(sample1+sample2);
+ }
+
+ public void track(Cell cell) {
+ int length = cell.getRowLength() + cell.getFamilyLength() + cell.getQualifierLength() + cell.getValueLength() + FIXED_CELL_LENGTH;
+ if (row == null || !CellUtil.matchingRow(cell, row)) {
+ incrRow(length);
+ row = cell;
+ col = cell;
+ } else if (col == null || !CellUtil.matchingFamily(cell, col) || !CellUtil.matchingColumn(cell, col)) {
+ incrCol(length);
+ col = cell;
+ }
+ curRowSize += length;
+ curColSize += length;
+ byteCount += length;
+ }
+
+ private void incrCol(int length) {
+ long oldMean = byteCount/colCount;
+ colCount++;
+
+ preColSizeVariance += (curColSize - oldMean)*(curColSize - (byteCount+length)/colCount);
+
+ maxColSize = Math.max(maxColSize, curColSize);
+ curColSize = 0;
+ }
+
+ private void incrRow(int length) {
+ long oldMean = byteCount/rowCount;
+ rowCount++;
+
+ preColSizeVariance += (curRowSize - oldMean)*(curRowSize - (byteCount+length)/rowCount);
+
+ maxRowSize = Math.max(maxRowSize, curRowSize);
+ curRowSize = 0;
+
+ incrCol(length);
+ }
+
+ public long getMaxColSize() {
+ return maxColSize;
+ }
+
+ public long getMaxRowSize() {
+ return maxRowSize;
+ }
+
+ public long getMeanRowSize() {
+ return byteCount/rowCount;
+ }
+
+ public long getMeanColSize() {
+ return byteCount/colCount;
+ }
+
+ public double getColSizeDeviation() {
+ return Math.sqrt(getColSizeVariance());
+ }
+
+ public double getRowSizeDeviation() {
+ return Math.sqrt(getRowSizeVariance());
+ }
+
+ public double getColSizeVariance() {
+ return colCount <= 1 ? 0 : preColSizeVariance/(colCount-1);
+ }
+
+ public double getRowSizeVariance() {
+ return rowCount <= 1 ? 0 : preRowSizeVariance/(rowCount-1);
+ }
+
+ // the following are needed for serialization into PB only
+ ////
+ public long getByteCount() {
+ return byteCount;
+ }
+
+ public long getRowCount() {
+ return rowCount;
+ }
+
+ public long getColCount() {
+ return colCount;
+ }
+
+ public double getPreColSizeVariance() {
+ return preColSizeVariance;
+ }
+
+ public double getPreRowSizeVariance() {
+ return preRowSizeVariance;
+ }
+ ////
+}
diff --git a/hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ZooKeeperScanPolicyObserver.java b/hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ZooKeeperScanPolicyObserver.java
index 8b6f975..6add713 100644
--- a/hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ZooKeeperScanPolicyObserver.java
+++ b/hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ZooKeeperScanPolicyObserver.java
@@ -39,7 +39,6 @@ import org.apache.hadoop.hbase.regionserver.ScanInfo;
import org.apache.hadoop.hbase.regionserver.InternalScanner;
import org.apache.hadoop.hbase.regionserver.KeyValueScanner;
import org.apache.hadoop.hbase.regionserver.ScanType;
-import org.apache.hadoop.hbase.regionserver.Store;
import org.apache.hadoop.hbase.regionserver.StoreScanner;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
@@ -189,7 +188,7 @@ public class ZooKeeperScanPolicyObserver extends BaseRegionObserver {
long ttl = Math.max(EnvironmentEdgeManager.currentTime() -
Bytes.toLong(data), oldSI.getTtl());
return new ScanInfo(store.getFamily(), ttl,
- oldSI.getTimeToPurgeDeletes(), oldSI.getComparator());
+ oldSI.getTimeToPurgeDeletes(), oldSI.getComparator(), null);
}
@Override
diff --git a/hbase-protocol/pom.xml b/hbase-protocol/pom.xml
index 297a7af..507b1b7 100644
--- a/hbase-protocol/pom.xml
+++ b/hbase-protocol/pom.xml
@@ -158,6 +158,7 @@
Aggregate.proto
Authentication.proto
Cell.proto
+ CellTracker.proto
Client.proto
ClusterId.proto
ClusterStatus.proto
diff --git a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/CellTrackerProtos.java b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/CellTrackerProtos.java
new file mode 100644
index 0000000..cb9faf7
--- /dev/null
+++ b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/CellTrackerProtos.java
@@ -0,0 +1,992 @@
+// Generated by the protocol buffer compiler. DO NOT EDIT!
+// source: CellTracker.proto
+
+package org.apache.hadoop.hbase.protobuf.generated;
+
+public final class CellTrackerProtos {
+ private CellTrackerProtos() {}
+ public static void registerAllExtensions(
+ com.google.protobuf.ExtensionRegistry registry) {
+ }
+ public interface CellStatsTrackerOrBuilder
+ extends com.google.protobuf.MessageOrBuilder {
+
+ // required int64 row_count = 1;
+ /**
+ * required int64 row_count = 1;
+ */
+ boolean hasRowCount();
+ /**
+ * required int64 row_count = 1;
+ */
+ long getRowCount();
+
+ // required int64 col_count = 2;
+ /**
+ * required int64 col_count = 2;
+ */
+ boolean hasColCount();
+ /**
+ * required int64 col_count = 2;
+ */
+ long getColCount();
+
+ // required int64 byte_count = 3;
+ /**
+ * required int64 byte_count = 3;
+ */
+ boolean hasByteCount();
+ /**
+ * required int64 byte_count = 3;
+ */
+ long getByteCount();
+
+ // required int64 max_col_size = 4;
+ /**
+ * required int64 max_col_size = 4;
+ */
+ boolean hasMaxColSize();
+ /**
+ * required int64 max_col_size = 4;
+ */
+ long getMaxColSize();
+
+ // required int64 max_row_size = 5;
+ /**
+ * required int64 max_row_size = 5;
+ */
+ boolean hasMaxRowSize();
+ /**
+ * required int64 max_row_size = 5;
+ */
+ long getMaxRowSize();
+
+ // required double pre_col_size_var = 6;
+ /**
+ * required double pre_col_size_var = 6;
+ */
+ boolean hasPreColSizeVar();
+ /**
+ * required double pre_col_size_var = 6;
+ */
+ double getPreColSizeVar();
+
+ // required double pre_row_size_var = 7;
+ /**
+ * required double pre_row_size_var = 7;
+ */
+ boolean hasPreRowSizeVar();
+ /**
+ * required double pre_row_size_var = 7;
+ */
+ double getPreRowSizeVar();
+ }
+ /**
+ * Protobuf type {@code CellStatsTracker}
+ */
+ public static final class CellStatsTracker extends
+ com.google.protobuf.GeneratedMessage
+ implements CellStatsTrackerOrBuilder {
+ // Use CellStatsTracker.newBuilder() to construct.
+ private CellStatsTracker(com.google.protobuf.GeneratedMessage.Builder> builder) {
+ super(builder);
+ this.unknownFields = builder.getUnknownFields();
+ }
+ private CellStatsTracker(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
+
+ private static final CellStatsTracker defaultInstance;
+ public static CellStatsTracker getDefaultInstance() {
+ return defaultInstance;
+ }
+
+ public CellStatsTracker getDefaultInstanceForType() {
+ return defaultInstance;
+ }
+
+ private final com.google.protobuf.UnknownFieldSet unknownFields;
+ @java.lang.Override
+ public final com.google.protobuf.UnknownFieldSet
+ getUnknownFields() {
+ return this.unknownFields;
+ }
+ private CellStatsTracker(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ initFields();
+ int mutable_bitField0_ = 0;
+ com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+ com.google.protobuf.UnknownFieldSet.newBuilder();
+ try {
+ boolean done = false;
+ while (!done) {
+ int tag = input.readTag();
+ switch (tag) {
+ case 0:
+ done = true;
+ break;
+ default: {
+ if (!parseUnknownField(input, unknownFields,
+ extensionRegistry, tag)) {
+ done = true;
+ }
+ break;
+ }
+ case 8: {
+ bitField0_ |= 0x00000001;
+ rowCount_ = input.readInt64();
+ break;
+ }
+ case 16: {
+ bitField0_ |= 0x00000002;
+ colCount_ = input.readInt64();
+ break;
+ }
+ case 24: {
+ bitField0_ |= 0x00000004;
+ byteCount_ = input.readInt64();
+ break;
+ }
+ case 32: {
+ bitField0_ |= 0x00000008;
+ maxColSize_ = input.readInt64();
+ break;
+ }
+ case 40: {
+ bitField0_ |= 0x00000010;
+ maxRowSize_ = input.readInt64();
+ break;
+ }
+ case 49: {
+ bitField0_ |= 0x00000020;
+ preColSizeVar_ = input.readDouble();
+ break;
+ }
+ case 57: {
+ bitField0_ |= 0x00000040;
+ preRowSizeVar_ = input.readDouble();
+ break;
+ }
+ }
+ }
+ } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+ throw e.setUnfinishedMessage(this);
+ } catch (java.io.IOException e) {
+ throw new com.google.protobuf.InvalidProtocolBufferException(
+ e.getMessage()).setUnfinishedMessage(this);
+ } finally {
+ this.unknownFields = unknownFields.build();
+ makeExtensionsImmutable();
+ }
+ }
+ public static final com.google.protobuf.Descriptors.Descriptor
+ getDescriptor() {
+ return org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.internal_static_CellStatsTracker_descriptor;
+ }
+
+ protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+ internalGetFieldAccessorTable() {
+ return org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.internal_static_CellStatsTracker_fieldAccessorTable
+ .ensureFieldAccessorsInitialized(
+ org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTracker.class, org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTracker.Builder.class);
+ }
+
+ public static com.google.protobuf.Parser PARSER =
+ new com.google.protobuf.AbstractParser() {
+ public CellStatsTracker parsePartialFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return new CellStatsTracker(input, extensionRegistry);
+ }
+ };
+
+ @java.lang.Override
+ public com.google.protobuf.Parser getParserForType() {
+ return PARSER;
+ }
+
+ private int bitField0_;
+ // required int64 row_count = 1;
+ public static final int ROW_COUNT_FIELD_NUMBER = 1;
+ private long rowCount_;
+ /**
+ * required int64 row_count = 1;
+ */
+ public boolean hasRowCount() {
+ return ((bitField0_ & 0x00000001) == 0x00000001);
+ }
+ /**
+ * required int64 row_count = 1;
+ */
+ public long getRowCount() {
+ return rowCount_;
+ }
+
+ // required int64 col_count = 2;
+ public static final int COL_COUNT_FIELD_NUMBER = 2;
+ private long colCount_;
+ /**
+ * required int64 col_count = 2;
+ */
+ public boolean hasColCount() {
+ return ((bitField0_ & 0x00000002) == 0x00000002);
+ }
+ /**
+ * required int64 col_count = 2;
+ */
+ public long getColCount() {
+ return colCount_;
+ }
+
+ // required int64 byte_count = 3;
+ public static final int BYTE_COUNT_FIELD_NUMBER = 3;
+ private long byteCount_;
+ /**
+ * required int64 byte_count = 3;
+ */
+ public boolean hasByteCount() {
+ return ((bitField0_ & 0x00000004) == 0x00000004);
+ }
+ /**
+ * required int64 byte_count = 3;
+ */
+ public long getByteCount() {
+ return byteCount_;
+ }
+
+ // required int64 max_col_size = 4;
+ public static final int MAX_COL_SIZE_FIELD_NUMBER = 4;
+ private long maxColSize_;
+ /**
+ * required int64 max_col_size = 4;
+ */
+ public boolean hasMaxColSize() {
+ return ((bitField0_ & 0x00000008) == 0x00000008);
+ }
+ /**
+ * required int64 max_col_size = 4;
+ */
+ public long getMaxColSize() {
+ return maxColSize_;
+ }
+
+ // required int64 max_row_size = 5;
+ public static final int MAX_ROW_SIZE_FIELD_NUMBER = 5;
+ private long maxRowSize_;
+ /**
+ * required int64 max_row_size = 5;
+ */
+ public boolean hasMaxRowSize() {
+ return ((bitField0_ & 0x00000010) == 0x00000010);
+ }
+ /**
+ * required int64 max_row_size = 5;
+ */
+ public long getMaxRowSize() {
+ return maxRowSize_;
+ }
+
+ // required double pre_col_size_var = 6;
+ public static final int PRE_COL_SIZE_VAR_FIELD_NUMBER = 6;
+ private double preColSizeVar_;
+ /**
+ * required double pre_col_size_var = 6;
+ */
+ public boolean hasPreColSizeVar() {
+ return ((bitField0_ & 0x00000020) == 0x00000020);
+ }
+ /**
+ * required double pre_col_size_var = 6;
+ */
+ public double getPreColSizeVar() {
+ return preColSizeVar_;
+ }
+
+ // required double pre_row_size_var = 7;
+ public static final int PRE_ROW_SIZE_VAR_FIELD_NUMBER = 7;
+ private double preRowSizeVar_;
+ /**
+ * required double pre_row_size_var = 7;
+ */
+ public boolean hasPreRowSizeVar() {
+ return ((bitField0_ & 0x00000040) == 0x00000040);
+ }
+ /**
+ * required double pre_row_size_var = 7;
+ */
+ public double getPreRowSizeVar() {
+ return preRowSizeVar_;
+ }
+
+ private void initFields() {
+ rowCount_ = 0L;
+ colCount_ = 0L;
+ byteCount_ = 0L;
+ maxColSize_ = 0L;
+ maxRowSize_ = 0L;
+ preColSizeVar_ = 0D;
+ preRowSizeVar_ = 0D;
+ }
+ private byte memoizedIsInitialized = -1;
+ public final boolean isInitialized() {
+ byte isInitialized = memoizedIsInitialized;
+ if (isInitialized != -1) return isInitialized == 1;
+
+ if (!hasRowCount()) {
+ memoizedIsInitialized = 0;
+ return false;
+ }
+ if (!hasColCount()) {
+ memoizedIsInitialized = 0;
+ return false;
+ }
+ if (!hasByteCount()) {
+ memoizedIsInitialized = 0;
+ return false;
+ }
+ if (!hasMaxColSize()) {
+ memoizedIsInitialized = 0;
+ return false;
+ }
+ if (!hasMaxRowSize()) {
+ memoizedIsInitialized = 0;
+ return false;
+ }
+ if (!hasPreColSizeVar()) {
+ memoizedIsInitialized = 0;
+ return false;
+ }
+ if (!hasPreRowSizeVar()) {
+ memoizedIsInitialized = 0;
+ return false;
+ }
+ memoizedIsInitialized = 1;
+ return true;
+ }
+
+ public void writeTo(com.google.protobuf.CodedOutputStream output)
+ throws java.io.IOException {
+ getSerializedSize();
+ if (((bitField0_ & 0x00000001) == 0x00000001)) {
+ output.writeInt64(1, rowCount_);
+ }
+ if (((bitField0_ & 0x00000002) == 0x00000002)) {
+ output.writeInt64(2, colCount_);
+ }
+ if (((bitField0_ & 0x00000004) == 0x00000004)) {
+ output.writeInt64(3, byteCount_);
+ }
+ if (((bitField0_ & 0x00000008) == 0x00000008)) {
+ output.writeInt64(4, maxColSize_);
+ }
+ if (((bitField0_ & 0x00000010) == 0x00000010)) {
+ output.writeInt64(5, maxRowSize_);
+ }
+ if (((bitField0_ & 0x00000020) == 0x00000020)) {
+ output.writeDouble(6, preColSizeVar_);
+ }
+ if (((bitField0_ & 0x00000040) == 0x00000040)) {
+ output.writeDouble(7, preRowSizeVar_);
+ }
+ getUnknownFields().writeTo(output);
+ }
+
+ private int memoizedSerializedSize = -1;
+ public int getSerializedSize() {
+ int size = memoizedSerializedSize;
+ if (size != -1) return size;
+
+ size = 0;
+ if (((bitField0_ & 0x00000001) == 0x00000001)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeInt64Size(1, rowCount_);
+ }
+ if (((bitField0_ & 0x00000002) == 0x00000002)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeInt64Size(2, colCount_);
+ }
+ if (((bitField0_ & 0x00000004) == 0x00000004)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeInt64Size(3, byteCount_);
+ }
+ if (((bitField0_ & 0x00000008) == 0x00000008)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeInt64Size(4, maxColSize_);
+ }
+ if (((bitField0_ & 0x00000010) == 0x00000010)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeInt64Size(5, maxRowSize_);
+ }
+ if (((bitField0_ & 0x00000020) == 0x00000020)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeDoubleSize(6, preColSizeVar_);
+ }
+ if (((bitField0_ & 0x00000040) == 0x00000040)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeDoubleSize(7, preRowSizeVar_);
+ }
+ size += getUnknownFields().getSerializedSize();
+ memoizedSerializedSize = size;
+ return size;
+ }
+
+ private static final long serialVersionUID = 0L;
+ @java.lang.Override
+ protected java.lang.Object writeReplace()
+ throws java.io.ObjectStreamException {
+ return super.writeReplace();
+ }
+
+ public static org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTracker parseFrom(
+ com.google.protobuf.ByteString data)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTracker parseFrom(
+ com.google.protobuf.ByteString data,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data, extensionRegistry);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTracker parseFrom(byte[] data)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTracker parseFrom(
+ byte[] data,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data, extensionRegistry);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTracker parseFrom(java.io.InputStream input)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTracker parseFrom(
+ java.io.InputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input, extensionRegistry);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTracker parseDelimitedFrom(java.io.InputStream input)
+ throws java.io.IOException {
+ return PARSER.parseDelimitedFrom(input);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTracker parseDelimitedFrom(
+ java.io.InputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseDelimitedFrom(input, extensionRegistry);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTracker parseFrom(
+ com.google.protobuf.CodedInputStream input)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTracker parseFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input, extensionRegistry);
+ }
+
+ public static Builder newBuilder() { return Builder.create(); }
+ public Builder newBuilderForType() { return newBuilder(); }
+ public static Builder newBuilder(org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTracker prototype) {
+ return newBuilder().mergeFrom(prototype);
+ }
+ public Builder toBuilder() { return newBuilder(this); }
+
+ @java.lang.Override
+ protected Builder newBuilderForType(
+ com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+ Builder builder = new Builder(parent);
+ return builder;
+ }
+ /**
+ * Protobuf type {@code CellStatsTracker}
+ */
+ public static final class Builder extends
+ com.google.protobuf.GeneratedMessage.Builder
+ implements org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTrackerOrBuilder {
+ public static final com.google.protobuf.Descriptors.Descriptor
+ getDescriptor() {
+ return org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.internal_static_CellStatsTracker_descriptor;
+ }
+
+ protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+ internalGetFieldAccessorTable() {
+ return org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.internal_static_CellStatsTracker_fieldAccessorTable
+ .ensureFieldAccessorsInitialized(
+ org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTracker.class, org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTracker.Builder.class);
+ }
+
+ // Construct using org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTracker.newBuilder()
+ private Builder() {
+ maybeForceBuilderInitialization();
+ }
+
+ private Builder(
+ com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+ super(parent);
+ maybeForceBuilderInitialization();
+ }
+ private void maybeForceBuilderInitialization() {
+ if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
+ }
+ }
+ private static Builder create() {
+ return new Builder();
+ }
+
+ public Builder clear() {
+ super.clear();
+ rowCount_ = 0L;
+ bitField0_ = (bitField0_ & ~0x00000001);
+ colCount_ = 0L;
+ bitField0_ = (bitField0_ & ~0x00000002);
+ byteCount_ = 0L;
+ bitField0_ = (bitField0_ & ~0x00000004);
+ maxColSize_ = 0L;
+ bitField0_ = (bitField0_ & ~0x00000008);
+ maxRowSize_ = 0L;
+ bitField0_ = (bitField0_ & ~0x00000010);
+ preColSizeVar_ = 0D;
+ bitField0_ = (bitField0_ & ~0x00000020);
+ preRowSizeVar_ = 0D;
+ bitField0_ = (bitField0_ & ~0x00000040);
+ return this;
+ }
+
+ public Builder clone() {
+ return create().mergeFrom(buildPartial());
+ }
+
+ public com.google.protobuf.Descriptors.Descriptor
+ getDescriptorForType() {
+ return org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.internal_static_CellStatsTracker_descriptor;
+ }
+
+ public org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTracker getDefaultInstanceForType() {
+ return org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTracker.getDefaultInstance();
+ }
+
+ public org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTracker build() {
+ org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTracker result = buildPartial();
+ if (!result.isInitialized()) {
+ throw newUninitializedMessageException(result);
+ }
+ return result;
+ }
+
+ public org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTracker buildPartial() {
+ org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTracker result = new org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTracker(this);
+ int from_bitField0_ = bitField0_;
+ int to_bitField0_ = 0;
+ if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+ to_bitField0_ |= 0x00000001;
+ }
+ result.rowCount_ = rowCount_;
+ if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+ to_bitField0_ |= 0x00000002;
+ }
+ result.colCount_ = colCount_;
+ if (((from_bitField0_ & 0x00000004) == 0x00000004)) {
+ to_bitField0_ |= 0x00000004;
+ }
+ result.byteCount_ = byteCount_;
+ if (((from_bitField0_ & 0x00000008) == 0x00000008)) {
+ to_bitField0_ |= 0x00000008;
+ }
+ result.maxColSize_ = maxColSize_;
+ if (((from_bitField0_ & 0x00000010) == 0x00000010)) {
+ to_bitField0_ |= 0x00000010;
+ }
+ result.maxRowSize_ = maxRowSize_;
+ if (((from_bitField0_ & 0x00000020) == 0x00000020)) {
+ to_bitField0_ |= 0x00000020;
+ }
+ result.preColSizeVar_ = preColSizeVar_;
+ if (((from_bitField0_ & 0x00000040) == 0x00000040)) {
+ to_bitField0_ |= 0x00000040;
+ }
+ result.preRowSizeVar_ = preRowSizeVar_;
+ result.bitField0_ = to_bitField0_;
+ onBuilt();
+ return result;
+ }
+
+ public Builder mergeFrom(com.google.protobuf.Message other) {
+ if (other instanceof org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTracker) {
+ return mergeFrom((org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTracker)other);
+ } else {
+ super.mergeFrom(other);
+ return this;
+ }
+ }
+
+ public Builder mergeFrom(org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTracker other) {
+ if (other == org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTracker.getDefaultInstance()) return this;
+ if (other.hasRowCount()) {
+ setRowCount(other.getRowCount());
+ }
+ if (other.hasColCount()) {
+ setColCount(other.getColCount());
+ }
+ if (other.hasByteCount()) {
+ setByteCount(other.getByteCount());
+ }
+ if (other.hasMaxColSize()) {
+ setMaxColSize(other.getMaxColSize());
+ }
+ if (other.hasMaxRowSize()) {
+ setMaxRowSize(other.getMaxRowSize());
+ }
+ if (other.hasPreColSizeVar()) {
+ setPreColSizeVar(other.getPreColSizeVar());
+ }
+ if (other.hasPreRowSizeVar()) {
+ setPreRowSizeVar(other.getPreRowSizeVar());
+ }
+ this.mergeUnknownFields(other.getUnknownFields());
+ return this;
+ }
+
+ public final boolean isInitialized() {
+ if (!hasRowCount()) {
+
+ return false;
+ }
+ if (!hasColCount()) {
+
+ return false;
+ }
+ if (!hasByteCount()) {
+
+ return false;
+ }
+ if (!hasMaxColSize()) {
+
+ return false;
+ }
+ if (!hasMaxRowSize()) {
+
+ return false;
+ }
+ if (!hasPreColSizeVar()) {
+
+ return false;
+ }
+ if (!hasPreRowSizeVar()) {
+
+ return false;
+ }
+ return true;
+ }
+
+ public Builder mergeFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTracker parsedMessage = null;
+ try {
+ parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry);
+ } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+ parsedMessage = (org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTracker) e.getUnfinishedMessage();
+ throw e;
+ } finally {
+ if (parsedMessage != null) {
+ mergeFrom(parsedMessage);
+ }
+ }
+ return this;
+ }
+ private int bitField0_;
+
+ // required int64 row_count = 1;
+ private long rowCount_ ;
+ /**
+ * required int64 row_count = 1;
+ */
+ public boolean hasRowCount() {
+ return ((bitField0_ & 0x00000001) == 0x00000001);
+ }
+ /**
+ * required int64 row_count = 1;
+ */
+ public long getRowCount() {
+ return rowCount_;
+ }
+ /**
+ * required int64 row_count = 1;
+ */
+ public Builder setRowCount(long value) {
+ bitField0_ |= 0x00000001;
+ rowCount_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * required int64 row_count = 1;
+ */
+ public Builder clearRowCount() {
+ bitField0_ = (bitField0_ & ~0x00000001);
+ rowCount_ = 0L;
+ onChanged();
+ return this;
+ }
+
+ // required int64 col_count = 2;
+ private long colCount_ ;
+ /**
+ * required int64 col_count = 2;
+ */
+ public boolean hasColCount() {
+ return ((bitField0_ & 0x00000002) == 0x00000002);
+ }
+ /**
+ * required int64 col_count = 2;
+ */
+ public long getColCount() {
+ return colCount_;
+ }
+ /**
+ * required int64 col_count = 2;
+ */
+ public Builder setColCount(long value) {
+ bitField0_ |= 0x00000002;
+ colCount_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * required int64 col_count = 2;
+ */
+ public Builder clearColCount() {
+ bitField0_ = (bitField0_ & ~0x00000002);
+ colCount_ = 0L;
+ onChanged();
+ return this;
+ }
+
+ // required int64 byte_count = 3;
+ private long byteCount_ ;
+ /**
+ * required int64 byte_count = 3;
+ */
+ public boolean hasByteCount() {
+ return ((bitField0_ & 0x00000004) == 0x00000004);
+ }
+ /**
+ * required int64 byte_count = 3;
+ */
+ public long getByteCount() {
+ return byteCount_;
+ }
+ /**
+ * required int64 byte_count = 3;
+ */
+ public Builder setByteCount(long value) {
+ bitField0_ |= 0x00000004;
+ byteCount_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * required int64 byte_count = 3;
+ */
+ public Builder clearByteCount() {
+ bitField0_ = (bitField0_ & ~0x00000004);
+ byteCount_ = 0L;
+ onChanged();
+ return this;
+ }
+
+ // required int64 max_col_size = 4;
+ private long maxColSize_ ;
+ /**
+ * required int64 max_col_size = 4;
+ */
+ public boolean hasMaxColSize() {
+ return ((bitField0_ & 0x00000008) == 0x00000008);
+ }
+ /**
+ * required int64 max_col_size = 4;
+ */
+ public long getMaxColSize() {
+ return maxColSize_;
+ }
+ /**
+ * required int64 max_col_size = 4;
+ */
+ public Builder setMaxColSize(long value) {
+ bitField0_ |= 0x00000008;
+ maxColSize_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * required int64 max_col_size = 4;
+ */
+ public Builder clearMaxColSize() {
+ bitField0_ = (bitField0_ & ~0x00000008);
+ maxColSize_ = 0L;
+ onChanged();
+ return this;
+ }
+
+ // required int64 max_row_size = 5;
+ private long maxRowSize_ ;
+ /**
+ * required int64 max_row_size = 5;
+ */
+ public boolean hasMaxRowSize() {
+ return ((bitField0_ & 0x00000010) == 0x00000010);
+ }
+ /**
+ * required int64 max_row_size = 5;
+ */
+ public long getMaxRowSize() {
+ return maxRowSize_;
+ }
+ /**
+ * required int64 max_row_size = 5;
+ */
+ public Builder setMaxRowSize(long value) {
+ bitField0_ |= 0x00000010;
+ maxRowSize_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * required int64 max_row_size = 5;
+ */
+ public Builder clearMaxRowSize() {
+ bitField0_ = (bitField0_ & ~0x00000010);
+ maxRowSize_ = 0L;
+ onChanged();
+ return this;
+ }
+
+ // required double pre_col_size_var = 6;
+ private double preColSizeVar_ ;
+ /**
+ * required double pre_col_size_var = 6;
+ */
+ public boolean hasPreColSizeVar() {
+ return ((bitField0_ & 0x00000020) == 0x00000020);
+ }
+ /**
+ * required double pre_col_size_var = 6;
+ */
+ public double getPreColSizeVar() {
+ return preColSizeVar_;
+ }
+ /**
+ * required double pre_col_size_var = 6;
+ */
+ public Builder setPreColSizeVar(double value) {
+ bitField0_ |= 0x00000020;
+ preColSizeVar_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * required double pre_col_size_var = 6;
+ */
+ public Builder clearPreColSizeVar() {
+ bitField0_ = (bitField0_ & ~0x00000020);
+ preColSizeVar_ = 0D;
+ onChanged();
+ return this;
+ }
+
+ // required double pre_row_size_var = 7;
+ private double preRowSizeVar_ ;
+ /**
+ * required double pre_row_size_var = 7;
+ */
+ public boolean hasPreRowSizeVar() {
+ return ((bitField0_ & 0x00000040) == 0x00000040);
+ }
+ /**
+ * required double pre_row_size_var = 7;
+ */
+ public double getPreRowSizeVar() {
+ return preRowSizeVar_;
+ }
+ /**
+ * required double pre_row_size_var = 7;
+ */
+ public Builder setPreRowSizeVar(double value) {
+ bitField0_ |= 0x00000040;
+ preRowSizeVar_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * required double pre_row_size_var = 7;
+ */
+ public Builder clearPreRowSizeVar() {
+ bitField0_ = (bitField0_ & ~0x00000040);
+ preRowSizeVar_ = 0D;
+ onChanged();
+ return this;
+ }
+
+ // @@protoc_insertion_point(builder_scope:CellStatsTracker)
+ }
+
+ static {
+ defaultInstance = new CellStatsTracker(true);
+ defaultInstance.initFields();
+ }
+
+ // @@protoc_insertion_point(class_scope:CellStatsTracker)
+ }
+
+ private static com.google.protobuf.Descriptors.Descriptor
+ internal_static_CellStatsTracker_descriptor;
+ private static
+ com.google.protobuf.GeneratedMessage.FieldAccessorTable
+ internal_static_CellStatsTracker_fieldAccessorTable;
+
+ public static com.google.protobuf.Descriptors.FileDescriptor
+ getDescriptor() {
+ return descriptor;
+ }
+ private static com.google.protobuf.Descriptors.FileDescriptor
+ descriptor;
+ static {
+ java.lang.String[] descriptorData = {
+ "\n\021CellTracker.proto\"\254\001\n\020CellStatsTracker" +
+ "\022\021\n\trow_count\030\001 \002(\003\022\021\n\tcol_count\030\002 \002(\003\022\022" +
+ "\n\nbyte_count\030\003 \002(\003\022\024\n\014max_col_size\030\004 \002(\003" +
+ "\022\024\n\014max_row_size\030\005 \002(\003\022\030\n\020pre_col_size_v" +
+ "ar\030\006 \002(\001\022\030\n\020pre_row_size_var\030\007 \002(\001B?\n*or" +
+ "g.apache.hadoop.hbase.protobuf.generated" +
+ "B\021CellTrackerProtos"
+ };
+ com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
+ new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
+ public com.google.protobuf.ExtensionRegistry assignDescriptors(
+ com.google.protobuf.Descriptors.FileDescriptor root) {
+ descriptor = root;
+ internal_static_CellStatsTracker_descriptor =
+ getDescriptor().getMessageTypes().get(0);
+ internal_static_CellStatsTracker_fieldAccessorTable = new
+ com.google.protobuf.GeneratedMessage.FieldAccessorTable(
+ internal_static_CellStatsTracker_descriptor,
+ new java.lang.String[] { "RowCount", "ColCount", "ByteCount", "MaxColSize", "MaxRowSize", "PreColSizeVar", "PreRowSizeVar", });
+ return null;
+ }
+ };
+ com.google.protobuf.Descriptors.FileDescriptor
+ .internalBuildGeneratedFileFrom(descriptorData,
+ new com.google.protobuf.Descriptors.FileDescriptor[] {
+ }, assigner);
+ }
+
+ // @@protoc_insertion_point(outer_class_scope)
+}
diff --git a/hbase-protocol/src/main/protobuf/CellTracker.proto b/hbase-protocol/src/main/protobuf/CellTracker.proto
new file mode 100644
index 0000000..2b7ae63
--- /dev/null
+++ b/hbase-protocol/src/main/protobuf/CellTracker.proto
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "CellTrackerProtos";
+
+ message CellStatsTracker {
+ required int64 row_count = 1;
+ required int64 col_count = 2;
+ required int64 byte_count = 3;
+ required int64 max_col_size = 4;
+ required int64 max_row_size = 5;
+ required double pre_col_size_var = 6;
+ required double pre_row_size_var = 7;
+}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
index 8b41401..a3baa43 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
@@ -63,6 +63,7 @@ import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.io.crypto.Cipher;
import org.apache.hadoop.hbase.io.crypto.Encryption;
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
+import org.apache.hadoop.hbase.io.hfile.CellStatsTracker;
import org.apache.hadoop.hbase.io.hfile.HFile;
import org.apache.hadoop.hbase.io.hfile.HFileContext;
import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
@@ -234,11 +235,6 @@ public class HStore implements Store {
Math.max(conf.getLong("hbase.hstore.time.to.purge.deletes", 0), 0);
LOG.trace("Time to purge deletes set to " + timeToPurgeDeletes +
"ms in store " + this);
- // Get TTL
- long ttl = determineTTLFromFamily(family);
- // Why not just pass a HColumnDescriptor in here altogether? Even if have
- // to clone it?
- scanInfo = new ScanInfo(family, ttl, timeToPurgeDeletes, this.comparator);
String className = conf.get(MEMSTORE_CLASS_NAME, DefaultMemStore.class.getName());
this.memstore = ReflectionUtils.instantiateWithCustomCtor(className, new Class[] {
Configuration.class, KeyValue.KVComparator.class }, new Object[] { conf, this.comparator });
@@ -267,6 +263,25 @@ public class HStore implements Store {
this.storeEngine = StoreEngine.create(this, this.conf, this.comparator);
this.storeEngine.getStoreFileManager().loadFiles(loadStoreFiles());
+ CellStatsTracker combined = null;
+ for (StoreFile sfile : getStorefiles()) {
+ byte[] pb = sfile.getReader().loadFileInfo().get(StoreFile.CELLSTATS_KEY);
+ if (pb != null) {
+ CellStatsTracker cst = ProtobufUtil.ToCellStatsTracker(org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTracker.parseFrom(pb));
+ if (combined == null) {
+ combined = cst;
+ } else {
+ combined.combine(cst);
+ }
+ }
+ }
+
+ // Get TTL
+ long ttl = determineTTLFromFamily(family);
+ // Why not just pass a HColumnDescriptor in here altogether? Even if have
+ // to clone it?
+ scanInfo = new ScanInfo(family, ttl, timeToPurgeDeletes, this.comparator, combined);
+
// Initialize checksum type from name. The names are CRC32, CRC32C, etc.
this.checksumType = getChecksumType(conf);
// initilize bytes per checksum
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanInfo.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanInfo.java
index a8b314e..6088006 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanInfo.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanInfo.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.hbase.regionserver;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.io.hfile.CellStatsTracker;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.KeepDeletedCells;
import org.apache.hadoop.hbase.KeyValue.KVComparator;
@@ -30,13 +31,14 @@ import org.apache.hadoop.hbase.util.ClassSize;
*/
@InterfaceAudience.Private
public class ScanInfo {
- private byte[] family;
- private int minVersions;
- private int maxVersions;
- private long ttl;
- private KeepDeletedCells keepDeletedCells;
- private long timeToPurgeDeletes;
- private KVComparator comparator;
+ private final byte[] family;
+ private final int minVersions;
+ private final int maxVersions;
+ private final long ttl;
+ private final KeepDeletedCells keepDeletedCells;
+ private final long timeToPurgeDeletes;
+ private final KVComparator comparator;
+ private final CellStatsTracker statsTracker;
public static final long FIXED_OVERHEAD = ClassSize.align(ClassSize.OBJECT
+ (2 * ClassSize.REFERENCE) + (2 * Bytes.SIZEOF_INT)
@@ -48,11 +50,12 @@ public class ScanInfo {
* @param timeToPurgeDeletes duration in ms after which a delete marker can
* be purged during a major compaction.
* @param comparator The store's comparator
+ * @param statsTracker Stats for store
*/
public ScanInfo(final HColumnDescriptor family, final long ttl, final long timeToPurgeDeletes,
- final KVComparator comparator) {
+ final KVComparator comparator, CellStatsTracker statsTracker) {
this(family.getName(), family.getMinVersions(), family.getMaxVersions(), ttl, family
- .getKeepDeletedCells(), timeToPurgeDeletes, comparator);
+ .getKeepDeletedCells(), timeToPurgeDeletes, comparator, statsTracker);
}
/**
@@ -60,14 +63,15 @@ public class ScanInfo {
* @param minVersions Store's MIN_VERSIONS setting
* @param maxVersions Store's VERSIONS setting
* @param ttl Store's TTL (in ms)
+ * @param keepDeletedCells Store's keepDeletedCells setting
* @param timeToPurgeDeletes duration in ms after which a delete marker can
* be purged during a major compaction.
- * @param keepDeletedCells Store's keepDeletedCells setting
* @param comparator The store's comparator
+ * @param statsTracker TODO
*/
public ScanInfo(final byte[] family, final int minVersions, final int maxVersions,
final long ttl, final KeepDeletedCells keepDeletedCells, final long timeToPurgeDeletes,
- final KVComparator comparator) {
+ final KVComparator comparator, CellStatsTracker statsTracker) {
this.family = family;
this.minVersions = minVersions;
this.maxVersions = maxVersions;
@@ -75,6 +79,7 @@ public class ScanInfo {
this.keepDeletedCells = keepDeletedCells;
this.timeToPurgeDeletes = timeToPurgeDeletes;
this.comparator = comparator;
+ this.statsTracker = statsTracker;
}
public byte[] getFamily() {
@@ -104,4 +109,8 @@ public class ScanInfo {
public KVComparator getComparator() {
return comparator;
}
+
+ public CellStatsTracker getStatsTracker() {
+ return statsTracker;
+ }
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
index 3e139a5..f99e244 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
@@ -48,10 +48,12 @@ import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
import org.apache.hadoop.hbase.io.hfile.BlockType;
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
+import org.apache.hadoop.hbase.io.hfile.CellStatsTracker;
import org.apache.hadoop.hbase.io.hfile.HFile;
import org.apache.hadoop.hbase.io.hfile.HFileContext;
import org.apache.hadoop.hbase.io.hfile.HFileScanner;
import org.apache.hadoop.hbase.io.hfile.HFileWriterV2;
+import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.regionserver.compactions.Compactor;
import org.apache.hadoop.hbase.util.BloomFilter;
import org.apache.hadoop.hbase.util.BloomFilterFactory;
@@ -109,6 +111,9 @@ public class StoreFile {
/** Key for Timerange information in metadata*/
public static final byte[] TIMERANGE_KEY = Bytes.toBytes("TIMERANGE");
+ /** Key for Cell Stats information in metadata*/
+ public static final byte[] CELLSTATS_KEY = Bytes.toBytes("CELLSTATS");
+
/** Key for timestamp of earliest-put in metadata*/
public static final byte[] EARLIEST_PUT_TS = Bytes.toBytes("EARLIEST_PUT_TS");
@@ -708,6 +713,7 @@ public class StoreFile {
protected int bytesPerChecksum;
TimeRangeTracker timeRangeTracker = new TimeRangeTracker();
+ CellStatsTracker cellStatsTracker = new CellStatsTracker();
/* isTimeRangeTrackerSet keeps track if the timeRange has already been set
* When flushing a memstore, we set TimeRange and use this variable to
* indicate that it doesn't need to be calculated again while
@@ -795,6 +801,7 @@ public class StoreFile {
public void appendTrackedTimestampsToMetadata() throws IOException {
appendFileInfo(TIMERANGE_KEY,WritableUtils.toByteArray(timeRangeTracker));
appendFileInfo(EARLIEST_PUT_TS, Bytes.toBytes(earliestPutTs));
+ appendFileInfo(CELLSTATS_KEY, ProtobufUtil.toCellStatsTracker(cellStatsTracker).toByteArray());
}
/**
@@ -820,6 +827,7 @@ public class StoreFile {
if (!isTimeRangeTrackerSet) {
timeRangeTracker.includeTimestamp(cell);
}
+ cellStatsTracker.track(cell);
}
private void appendGeneralBloomfilter(final Cell cell) throws IOException {
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/NoOpScanPolicyObserver.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/NoOpScanPolicyObserver.java
index 36493cd..42989d2 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/NoOpScanPolicyObserver.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/NoOpScanPolicyObserver.java
@@ -46,7 +46,7 @@ public class NoOpScanPolicyObserver extends BaseRegionObserver {
Store store, KeyValueScanner memstoreScanner, InternalScanner s) throws IOException {
ScanInfo oldSI = store.getScanInfo();
ScanInfo scanInfo = new ScanInfo(store.getFamily(), oldSI.getTtl(),
- oldSI.getTimeToPurgeDeletes(), oldSI.getComparator());
+ oldSI.getTimeToPurgeDeletes(), oldSI.getComparator(), null);
Scan scan = new Scan();
scan.setMaxVersions(oldSI.getMaxVersions());
return new StoreScanner(store, scanInfo, scan, Collections.singletonList(memstoreScanner),
@@ -63,7 +63,7 @@ public class NoOpScanPolicyObserver extends BaseRegionObserver {
// this demonstrates how to override the scanners default behavior
ScanInfo oldSI = store.getScanInfo();
ScanInfo scanInfo = new ScanInfo(store.getFamily(), oldSI.getTtl(),
- oldSI.getTimeToPurgeDeletes(), oldSI.getComparator());
+ oldSI.getTimeToPurgeDeletes(), oldSI.getComparator(), null);
Scan scan = new Scan();
scan.setMaxVersions(oldSI.getMaxVersions());
return new StoreScanner(store, scanInfo, scan, scanners, scanType,
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
index 4900ef4..a982c47 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
@@ -193,7 +193,7 @@ public class TestCompaction {
ScanInfo old = store.getScanInfo();
ScanInfo si = new ScanInfo(old.getFamily(),
old.getMinVersions(), old.getMaxVersions(), ttl,
- old.getKeepDeletedCells(), 0, old.getComparator());
+ old.getKeepDeletedCells(), 0, old.getComparator(), null);
store.setScanInfo(si);
}
Thread.sleep(ttl);
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultCompactSelection.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultCompactSelection.java
index a78e1d5..d027caa 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultCompactSelection.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultCompactSelection.java
@@ -312,7 +312,7 @@ public class TestDefaultCompactSelection extends TestCase {
ScanInfo newScanInfo = new ScanInfo(oldScanInfo.getFamily(),
oldScanInfo.getMinVersions(), oldScanInfo.getMaxVersions(), 600,
oldScanInfo.getKeepDeletedCells(), oldScanInfo.getTimeToPurgeDeletes(),
- oldScanInfo.getComparator());
+ oldScanInfo.getComparator(), null);
store.setScanInfo(newScanInfo);
// Do not compact empty store file
List candidates = sfCreate(0);
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java
index d6d82df..0716b0c 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java
@@ -96,7 +96,7 @@ public class TestDefaultMemStore extends TestCase {
List result = new ArrayList| ();
ScanInfo scanInfo =
new ScanInfo(null, 0, 1, HConstants.LATEST_TIMESTAMP, KeepDeletedCells.FALSE, 0,
- this.memstore.comparator);
+ this.memstore.comparator, null);
ScanType scanType = ScanType.USER_SCAN;
StoreScanner s = new StoreScanner(scan, scanInfo, scanType, null, memstorescanners);
int count = 0;
@@ -516,7 +516,7 @@ public class TestDefaultMemStore extends TestCase {
//starting from each row, validate results should contain the starting row
for (int startRowId = 0; startRowId < ROW_COUNT; startRowId++) {
ScanInfo scanInfo = new ScanInfo(FAMILY, 0, 1, Integer.MAX_VALUE, KeepDeletedCells.FALSE,
- 0, this.memstore.comparator);
+ 0, this.memstore.comparator, null);
ScanType scanType = ScanType.USER_SCAN;
InternalScanner scanner = new StoreScanner(new Scan(
Bytes.toBytes(startRowId)), scanInfo, scanType, null,
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java
index 0469f0e..48fa52b 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java
@@ -290,7 +290,7 @@ public class TestMajorCompaction {
ScanInfo old = store.getScanInfo();
ScanInfo si = new ScanInfo(old.getFamily(),
old.getMinVersions(), old.getMaxVersions(), ttl,
- old.getKeepDeletedCells(), 0, old.getComparator());
+ old.getKeepDeletedCells(), 0, old.getComparator(), null);
store.setScanInfo(si);
}
Thread.sleep(1000);
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java
index 97c66e4..af3f116 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java
@@ -98,7 +98,7 @@ public class TestQueryMatcher extends HBaseTestCase {
private void _testMatch_ExplicitColumns(Scan scan, List expected) throws IOException {
// 2,4,5
ScanQueryMatcher qm = new ScanQueryMatcher(scan, new ScanInfo(fam2,
- 0, 1, ttl, KeepDeletedCells.FALSE, 0, rowComparator), get.getFamilyMap().get(fam2),
+ 0, 1, ttl, KeepDeletedCells.FALSE, 0, rowComparator, null), get.getFamilyMap().get(fam2),
EnvironmentEdgeManager.currentTime() - ttl);
List memstore = new ArrayList();
@@ -183,7 +183,7 @@ public class TestQueryMatcher extends HBaseTestCase {
expected.add(ScanQueryMatcher.MatchCode.DONE);
ScanQueryMatcher qm = new ScanQueryMatcher(scan, new ScanInfo(fam2,
- 0, 1, ttl, KeepDeletedCells.FALSE, 0, rowComparator), null,
+ 0, 1, ttl, KeepDeletedCells.FALSE, 0, rowComparator, null), null,
EnvironmentEdgeManager.currentTime() - ttl);
List memstore = new ArrayList();
@@ -239,7 +239,7 @@ public class TestQueryMatcher extends HBaseTestCase {
long now = EnvironmentEdgeManager.currentTime();
ScanQueryMatcher qm =
new ScanQueryMatcher(scan, new ScanInfo(fam2, 0, 1, testTTL, KeepDeletedCells.FALSE, 0,
- rowComparator), get.getFamilyMap().get(fam2), now - testTTL);
+ rowComparator, null), get.getFamilyMap().get(fam2), now - testTTL);
KeyValue [] kvs = new KeyValue[] {
new KeyValue(row1, fam2, col1, now-100, data),
@@ -293,7 +293,7 @@ public class TestQueryMatcher extends HBaseTestCase {
long now = EnvironmentEdgeManager.currentTime();
ScanQueryMatcher qm = new ScanQueryMatcher(scan, new ScanInfo(fam2,
- 0, 1, testTTL, KeepDeletedCells.FALSE, 0, rowComparator), null,
+ 0, 1, testTTL, KeepDeletedCells.FALSE, 0, rowComparator, null), null,
now - testTTL);
KeyValue [] kvs = new KeyValue[] {
@@ -349,7 +349,7 @@ public class TestQueryMatcher extends HBaseTestCase {
byte[] from, byte[] to, byte[][] rows, MatchCode... expected) throws IOException {
long now = EnvironmentEdgeManager.currentTime();
// Set time to purge deletes to negative value to avoid it ever happening.
- ScanInfo scanInfo = new ScanInfo(fam2, 0, 1, ttl, KeepDeletedCells.FALSE, -1L, rowComparator);
+ ScanInfo scanInfo = new ScanInfo(fam2, 0, 1, ttl, KeepDeletedCells.FALSE, -1L, rowComparator, null);
NavigableSet cols = get.getFamilyMap().get(fam2);
ScanQueryMatcher qm = new ScanQueryMatcher(scan, scanInfo, cols, Long.MAX_VALUE,
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestReversibleScanners.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestReversibleScanners.java
index 5a95df11..0c42ac1 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestReversibleScanners.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestReversibleScanners.java
@@ -253,7 +253,7 @@ public class TestReversibleScanners {
ScanType scanType = ScanType.USER_SCAN;
ScanInfo scanInfo = new ScanInfo(FAMILYNAME, 0, Integer.MAX_VALUE,
- Long.MAX_VALUE, KeepDeletedCells.FALSE, 0, KeyValue.COMPARATOR);
+ Long.MAX_VALUE, KeepDeletedCells.FALSE, 0, KeyValue.COMPARATOR, null);
// Case 1.Test a full reversed scan
Scan scan = new Scan();
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java
index bf9fed6..1782482 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java
@@ -49,7 +49,7 @@ public class TestStoreScanner extends TestCase {
private static final String CF_STR = "cf";
final byte [] CF = Bytes.toBytes(CF_STR);
private ScanInfo scanInfo = new ScanInfo(CF, 0, Integer.MAX_VALUE,
- Long.MAX_VALUE, KeepDeletedCells.FALSE, 0, KeyValue.COMPARATOR);
+ Long.MAX_VALUE, KeepDeletedCells.FALSE, 0, KeyValue.COMPARATOR, null);
private ScanType scanType = ScanType.USER_SCAN;
public void setUp() throws Exception {
@@ -418,7 +418,7 @@ public class TestStoreScanner extends TestCase {
Scan scan = new Scan();
scan.setMaxVersions(1);
ScanInfo scanInfo = new ScanInfo(CF, 0, 1, 500, KeepDeletedCells.FALSE, 0,
- KeyValue.COMPARATOR);
+ KeyValue.COMPARATOR, null);
ScanType scanType = ScanType.USER_SCAN;
StoreScanner scanner =
new StoreScanner(scan, scanInfo, scanType,
@@ -489,7 +489,7 @@ public class TestStoreScanner extends TestCase {
scan.setMaxVersions(1);
// scanner with ttl equal to 500
ScanInfo scanInfo = new ScanInfo(CF, 0, 1, 500, KeepDeletedCells.FALSE, 0,
- KeyValue.COMPARATOR);
+ KeyValue.COMPARATOR, null);
ScanType scanType = ScanType.USER_SCAN;
StoreScanner scanner =
new StoreScanner(scan, scanInfo, scanType, null, scanners);
@@ -553,7 +553,7 @@ public class TestStoreScanner extends TestCase {
2 /* maxVersions */, 500 /* ttl */,
KeepDeletedCells.FALSE /* keepDeletedCells */,
200, /* timeToPurgeDeletes */
- KeyValue.COMPARATOR);
+ KeyValue.COMPARATOR, null);
StoreScanner scanner =
new StoreScanner(scan, scanInfo,
ScanType.COMPACT_DROP_DELETES, null, scanners,
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStripeCompactor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStripeCompactor.java
index ed8b819..4effdbb 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStripeCompactor.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStripeCompactor.java
@@ -176,7 +176,7 @@ public class TestStripeCompactor {
// Create store mock that is satisfactory for compactor.
HColumnDescriptor col = new HColumnDescriptor(NAME_OF_THINGS);
- ScanInfo si = new ScanInfo(col, Long.MAX_VALUE, 0, new KVComparator());
+ ScanInfo si = new ScanInfo(col, Long.MAX_VALUE, 0, new KVComparator(), null);
Store store = mock(Store.class);
when(store.getFamily()).thenReturn(col);
when(store.getScanInfo()).thenReturn(si);
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java
index 034d6bc..9ba16ed 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java
@@ -252,7 +252,7 @@ public class TestCoprocessorScanPolicy {
ScanInfo scanInfo = new ScanInfo(family.getName(), family.getMinVersions(),
newVersions == null ? family.getMaxVersions() : newVersions,
newTtl == null ? oldSI.getTtl() : newTtl, family.getKeepDeletedCells(),
- oldSI.getTimeToPurgeDeletes(), oldSI.getComparator());
+ oldSI.getTimeToPurgeDeletes(), oldSI.getComparator(), null);
Scan scan = new Scan();
scan.setMaxVersions(newVersions == null ? oldSI.getMaxVersions() : newVersions);
return new StoreScanner(store, scanInfo, scan, Collections.singletonList(memstoreScanner),
@@ -272,7 +272,7 @@ public class TestCoprocessorScanPolicy {
ScanInfo scanInfo = new ScanInfo(family.getName(), family.getMinVersions(),
newVersions == null ? family.getMaxVersions() : newVersions,
newTtl == null ? oldSI.getTtl() : newTtl, family.getKeepDeletedCells(),
- oldSI.getTimeToPurgeDeletes(), oldSI.getComparator());
+ oldSI.getTimeToPurgeDeletes(), oldSI.getComparator(), null);
Scan scan = new Scan();
scan.setMaxVersions(newVersions == null ? oldSI.getMaxVersions() : newVersions);
return new StoreScanner(store, scanInfo, scan, scanners, scanType,
@@ -292,7 +292,7 @@ public class TestCoprocessorScanPolicy {
ScanInfo scanInfo = new ScanInfo(family.getName(), family.getMinVersions(),
newVersions == null ? family.getMaxVersions() : newVersions,
newTtl == null ? oldSI.getTtl() : newTtl, family.getKeepDeletedCells(),
- oldSI.getTimeToPurgeDeletes(), oldSI.getComparator());
+ oldSI.getTimeToPurgeDeletes(), oldSI.getComparator(), null);
return new StoreScanner(store, scanInfo, scan, targetCols,
((HStore) store).getHRegion().getReadpoint(IsolationLevel.READ_COMMITTED));
} else {
| |