diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java index 768da76..409fc89 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java @@ -67,6 +67,7 @@ import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.filter.ByteArrayComparable; import org.apache.hadoop.hbase.filter.Filter; import org.apache.hadoop.hbase.io.TimeRange; +import org.apache.hadoop.hbase.io.hfile.CellStatsTracker; import org.apache.hadoop.hbase.protobuf.generated.AccessControlProtos; import org.apache.hadoop.hbase.protobuf.generated.AccessControlProtos.AccessControlService; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService; @@ -86,6 +87,7 @@ import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ServerInfo; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.SplitRegionRequest; import org.apache.hadoop.hbase.protobuf.generated.AuthenticationProtos; import org.apache.hadoop.hbase.protobuf.generated.CellProtos; +import org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos; import org.apache.hadoop.hbase.protobuf.generated.ClientProtos; import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.BulkLoadHFileRequest; import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.BulkLoadHFileResponse; @@ -2964,4 +2966,23 @@ public final class ProtobufUtil { .setScope(toProtoQuotaScope(scope)) .build(); } + + public static CellStatsTracker ToCellStatsTracker(CellTrackerProtos.CellStatsTracker proto) + throws IOException { + return new CellStatsTracker(proto.getColCount(), proto.getRowCount(), proto.getByteCount(), + proto.getMaxColSize(), proto.getMaxRowSize(), proto.getPreColSizeVar(), + proto.getPreRowSizeVar()); + } + + public static CellTrackerProtos.CellStatsTracker toCellStatsTracker(CellStatsTracker tracker) + throws IOException { + CellTrackerProtos.CellStatsTracker.Builder builder = + CellTrackerProtos.CellStatsTracker.newBuilder(); + builder.setByteCount(tracker.getByteCount()).setColCount(tracker.getColCount()) + .setRowCount(tracker.getRowCount()).setMaxColSize(tracker.getMaxColSize()) + .setMaxRowSize(tracker.getMaxRowSize()).setPreColSizeVar(tracker.getPreColSizeVariance()) + .setPreRowSizeVar(tracker.getPreRowSizeVariance()); + return builder.build(); + } + } diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/hfile/CellStatsTracker.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/hfile/CellStatsTracker.java new file mode 100644 index 0000000..517f249 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/hfile/CellStatsTracker.java @@ -0,0 +1,145 @@ +package org.apache.hadoop.hbase.io.hfile; + +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; + +public class CellStatsTracker { + private Cell row = null; + private Cell col = null; + + private long colCount; + private long rowCount; + private long byteCount; + + private long curColSize = 0; + private long curRowSize = 0; + + private long maxColSize; + private long maxRowSize; + + private double preColSizeVariance; + private double preRowSizeVariance; + + private int FIXED_CELL_LENGTH = 16; // is this right? + + public CellStatsTracker() {} + + // used to deserialize from PB + public CellStatsTracker(long colCount, long rowCount, long byteCount, long maxColSize, long maxRowSize, double preColSizeVariance, double preRowSizeVariance) { + this.colCount = colCount; + this.rowCount = rowCount; + this.byteCount = byteCount; + this.maxColSize = maxColSize; + this.maxRowSize = maxRowSize; + this.preColSizeVariance = preColSizeVariance; + this.preRowSizeVariance = preRowSizeVariance; + } + + public void combine(CellStatsTracker other) { + this.colCount += other.colCount; + this.rowCount += other.rowCount; + this.byteCount += other.byteCount; + this.maxColSize = Math.max(maxColSize, other.maxColSize); + this.maxRowSize = Math.max(maxRowSize, other.maxRowSize); + this.preColSizeVariance = combinePreVariance(this.getColSizeVariance(), this.getColCount(), other.getColSizeVariance(), other.getColCount()); + this.preRowSizeVariance = combinePreVariance(this.getRowSizeVariance(), this.getRowCount(), other.getRowSizeVariance(), other.getRowCount()); + } + + private double combinePreVariance(double v1, long sample1, double v2, long sample2) { + if (sample1 == 0) return v2; + if (sample2 == 0) return v1; + return (v1/sample1+v2/sample2)*(sample1+sample2); + } + + public void track(Cell cell) { + int length = cell.getRowLength() + cell.getFamilyLength() + cell.getQualifierLength() + cell.getValueLength() + FIXED_CELL_LENGTH; + if (row == null || !CellUtil.matchingRow(cell, row)) { + incrRow(length); + row = cell; + col = cell; + } else if (col == null || !CellUtil.matchingFamily(cell, col) || !CellUtil.matchingColumn(cell, col)) { + incrCol(length); + col = cell; + } + curRowSize += length; + curColSize += length; + byteCount += length; + } + + private void incrCol(int length) { + long oldMean = byteCount/colCount; + colCount++; + + preColSizeVariance += (curColSize - oldMean)*(curColSize - (byteCount+length)/colCount); + + maxColSize = Math.max(maxColSize, curColSize); + curColSize = 0; + } + + private void incrRow(int length) { + long oldMean = byteCount/rowCount; + rowCount++; + + preColSizeVariance += (curRowSize - oldMean)*(curRowSize - (byteCount+length)/rowCount); + + maxRowSize = Math.max(maxRowSize, curRowSize); + curRowSize = 0; + + incrCol(length); + } + + public long getMaxColSize() { + return maxColSize; + } + + public long getMaxRowSize() { + return maxRowSize; + } + + public long getMeanRowSize() { + return byteCount/rowCount; + } + + public long getMeanColSize() { + return byteCount/colCount; + } + + public double getColSizeDeviation() { + return Math.sqrt(getColSizeVariance()); + } + + public double getRowSizeDeviation() { + return Math.sqrt(getRowSizeVariance()); + } + + public double getColSizeVariance() { + return colCount <= 1 ? 0 : preColSizeVariance/(colCount-1); + } + + public double getRowSizeVariance() { + return rowCount <= 1 ? 0 : preRowSizeVariance/(rowCount-1); + } + + // the following are needed for serialization into PB only + //// + public long getByteCount() { + return byteCount; + } + + public long getRowCount() { + return rowCount; + } + + public long getColCount() { + return colCount; + } + + public double getPreColSizeVariance() { + return preColSizeVariance; + } + + public double getPreRowSizeVariance() { + return preRowSizeVariance; + } + //// +} diff --git a/hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ZooKeeperScanPolicyObserver.java b/hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ZooKeeperScanPolicyObserver.java index 8b6f975..6add713 100644 --- a/hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ZooKeeperScanPolicyObserver.java +++ b/hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ZooKeeperScanPolicyObserver.java @@ -39,7 +39,6 @@ import org.apache.hadoop.hbase.regionserver.ScanInfo; import org.apache.hadoop.hbase.regionserver.InternalScanner; import org.apache.hadoop.hbase.regionserver.KeyValueScanner; import org.apache.hadoop.hbase.regionserver.ScanType; -import org.apache.hadoop.hbase.regionserver.Store; import org.apache.hadoop.hbase.regionserver.StoreScanner; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; @@ -189,7 +188,7 @@ public class ZooKeeperScanPolicyObserver extends BaseRegionObserver { long ttl = Math.max(EnvironmentEdgeManager.currentTime() - Bytes.toLong(data), oldSI.getTtl()); return new ScanInfo(store.getFamily(), ttl, - oldSI.getTimeToPurgeDeletes(), oldSI.getComparator()); + oldSI.getTimeToPurgeDeletes(), oldSI.getComparator(), null); } @Override diff --git a/hbase-protocol/pom.xml b/hbase-protocol/pom.xml index 297a7af..507b1b7 100644 --- a/hbase-protocol/pom.xml +++ b/hbase-protocol/pom.xml @@ -158,6 +158,7 @@ Aggregate.proto Authentication.proto Cell.proto + CellTracker.proto Client.proto ClusterId.proto ClusterStatus.proto diff --git a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/CellTrackerProtos.java b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/CellTrackerProtos.java new file mode 100644 index 0000000..cb9faf7 --- /dev/null +++ b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/CellTrackerProtos.java @@ -0,0 +1,992 @@ +// Generated by the protocol buffer compiler. DO NOT EDIT! +// source: CellTracker.proto + +package org.apache.hadoop.hbase.protobuf.generated; + +public final class CellTrackerProtos { + private CellTrackerProtos() {} + public static void registerAllExtensions( + com.google.protobuf.ExtensionRegistry registry) { + } + public interface CellStatsTrackerOrBuilder + extends com.google.protobuf.MessageOrBuilder { + + // required int64 row_count = 1; + /** + * required int64 row_count = 1; + */ + boolean hasRowCount(); + /** + * required int64 row_count = 1; + */ + long getRowCount(); + + // required int64 col_count = 2; + /** + * required int64 col_count = 2; + */ + boolean hasColCount(); + /** + * required int64 col_count = 2; + */ + long getColCount(); + + // required int64 byte_count = 3; + /** + * required int64 byte_count = 3; + */ + boolean hasByteCount(); + /** + * required int64 byte_count = 3; + */ + long getByteCount(); + + // required int64 max_col_size = 4; + /** + * required int64 max_col_size = 4; + */ + boolean hasMaxColSize(); + /** + * required int64 max_col_size = 4; + */ + long getMaxColSize(); + + // required int64 max_row_size = 5; + /** + * required int64 max_row_size = 5; + */ + boolean hasMaxRowSize(); + /** + * required int64 max_row_size = 5; + */ + long getMaxRowSize(); + + // required double pre_col_size_var = 6; + /** + * required double pre_col_size_var = 6; + */ + boolean hasPreColSizeVar(); + /** + * required double pre_col_size_var = 6; + */ + double getPreColSizeVar(); + + // required double pre_row_size_var = 7; + /** + * required double pre_row_size_var = 7; + */ + boolean hasPreRowSizeVar(); + /** + * required double pre_row_size_var = 7; + */ + double getPreRowSizeVar(); + } + /** + * Protobuf type {@code CellStatsTracker} + */ + public static final class CellStatsTracker extends + com.google.protobuf.GeneratedMessage + implements CellStatsTrackerOrBuilder { + // Use CellStatsTracker.newBuilder() to construct. + private CellStatsTracker(com.google.protobuf.GeneratedMessage.Builder builder) { + super(builder); + this.unknownFields = builder.getUnknownFields(); + } + private CellStatsTracker(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); } + + private static final CellStatsTracker defaultInstance; + public static CellStatsTracker getDefaultInstance() { + return defaultInstance; + } + + public CellStatsTracker getDefaultInstanceForType() { + return defaultInstance; + } + + private final com.google.protobuf.UnknownFieldSet unknownFields; + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet + getUnknownFields() { + return this.unknownFields; + } + private CellStatsTracker( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + initFields(); + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, + extensionRegistry, tag)) { + done = true; + } + break; + } + case 8: { + bitField0_ |= 0x00000001; + rowCount_ = input.readInt64(); + break; + } + case 16: { + bitField0_ |= 0x00000002; + colCount_ = input.readInt64(); + break; + } + case 24: { + bitField0_ |= 0x00000004; + byteCount_ = input.readInt64(); + break; + } + case 32: { + bitField0_ |= 0x00000008; + maxColSize_ = input.readInt64(); + break; + } + case 40: { + bitField0_ |= 0x00000010; + maxRowSize_ = input.readInt64(); + break; + } + case 49: { + bitField0_ |= 0x00000020; + preColSizeVar_ = input.readDouble(); + break; + } + case 57: { + bitField0_ |= 0x00000040; + preRowSizeVar_ = input.readDouble(); + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException( + e.getMessage()).setUnfinishedMessage(this); + } finally { + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.internal_static_CellStatsTracker_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.internal_static_CellStatsTracker_fieldAccessorTable + .ensureFieldAccessorsInitialized( + org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTracker.class, org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTracker.Builder.class); + } + + public static com.google.protobuf.Parser PARSER = + new com.google.protobuf.AbstractParser() { + public CellStatsTracker parsePartialFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new CellStatsTracker(input, extensionRegistry); + } + }; + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + private int bitField0_; + // required int64 row_count = 1; + public static final int ROW_COUNT_FIELD_NUMBER = 1; + private long rowCount_; + /** + * required int64 row_count = 1; + */ + public boolean hasRowCount() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + /** + * required int64 row_count = 1; + */ + public long getRowCount() { + return rowCount_; + } + + // required int64 col_count = 2; + public static final int COL_COUNT_FIELD_NUMBER = 2; + private long colCount_; + /** + * required int64 col_count = 2; + */ + public boolean hasColCount() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + /** + * required int64 col_count = 2; + */ + public long getColCount() { + return colCount_; + } + + // required int64 byte_count = 3; + public static final int BYTE_COUNT_FIELD_NUMBER = 3; + private long byteCount_; + /** + * required int64 byte_count = 3; + */ + public boolean hasByteCount() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + /** + * required int64 byte_count = 3; + */ + public long getByteCount() { + return byteCount_; + } + + // required int64 max_col_size = 4; + public static final int MAX_COL_SIZE_FIELD_NUMBER = 4; + private long maxColSize_; + /** + * required int64 max_col_size = 4; + */ + public boolean hasMaxColSize() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + /** + * required int64 max_col_size = 4; + */ + public long getMaxColSize() { + return maxColSize_; + } + + // required int64 max_row_size = 5; + public static final int MAX_ROW_SIZE_FIELD_NUMBER = 5; + private long maxRowSize_; + /** + * required int64 max_row_size = 5; + */ + public boolean hasMaxRowSize() { + return ((bitField0_ & 0x00000010) == 0x00000010); + } + /** + * required int64 max_row_size = 5; + */ + public long getMaxRowSize() { + return maxRowSize_; + } + + // required double pre_col_size_var = 6; + public static final int PRE_COL_SIZE_VAR_FIELD_NUMBER = 6; + private double preColSizeVar_; + /** + * required double pre_col_size_var = 6; + */ + public boolean hasPreColSizeVar() { + return ((bitField0_ & 0x00000020) == 0x00000020); + } + /** + * required double pre_col_size_var = 6; + */ + public double getPreColSizeVar() { + return preColSizeVar_; + } + + // required double pre_row_size_var = 7; + public static final int PRE_ROW_SIZE_VAR_FIELD_NUMBER = 7; + private double preRowSizeVar_; + /** + * required double pre_row_size_var = 7; + */ + public boolean hasPreRowSizeVar() { + return ((bitField0_ & 0x00000040) == 0x00000040); + } + /** + * required double pre_row_size_var = 7; + */ + public double getPreRowSizeVar() { + return preRowSizeVar_; + } + + private void initFields() { + rowCount_ = 0L; + colCount_ = 0L; + byteCount_ = 0L; + maxColSize_ = 0L; + maxRowSize_ = 0L; + preColSizeVar_ = 0D; + preRowSizeVar_ = 0D; + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + if (!hasRowCount()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasColCount()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasByteCount()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasMaxColSize()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasMaxRowSize()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasPreColSizeVar()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasPreRowSizeVar()) { + memoizedIsInitialized = 0; + return false; + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeInt64(1, rowCount_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + output.writeInt64(2, colCount_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + output.writeInt64(3, byteCount_); + } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + output.writeInt64(4, maxColSize_); + } + if (((bitField0_ & 0x00000010) == 0x00000010)) { + output.writeInt64(5, maxRowSize_); + } + if (((bitField0_ & 0x00000020) == 0x00000020)) { + output.writeDouble(6, preColSizeVar_); + } + if (((bitField0_ & 0x00000040) == 0x00000040)) { + output.writeDouble(7, preRowSizeVar_); + } + getUnknownFields().writeTo(output); + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream + .computeInt64Size(1, rowCount_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.CodedOutputStream + .computeInt64Size(2, colCount_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + size += com.google.protobuf.CodedOutputStream + .computeInt64Size(3, byteCount_); + } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + size += com.google.protobuf.CodedOutputStream + .computeInt64Size(4, maxColSize_); + } + if (((bitField0_ & 0x00000010) == 0x00000010)) { + size += com.google.protobuf.CodedOutputStream + .computeInt64Size(5, maxRowSize_); + } + if (((bitField0_ & 0x00000020) == 0x00000020)) { + size += com.google.protobuf.CodedOutputStream + .computeDoubleSize(6, preColSizeVar_); + } + if (((bitField0_ & 0x00000040) == 0x00000040)) { + size += com.google.protobuf.CodedOutputStream + .computeDoubleSize(7, preRowSizeVar_); + } + size += getUnknownFields().getSerializedSize(); + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + public static org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTracker parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTracker parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTracker parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTracker parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTracker parseFrom(java.io.InputStream input) + throws java.io.IOException { + return PARSER.parseFrom(input); + } + public static org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTracker parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return PARSER.parseFrom(input, extensionRegistry); + } + public static org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTracker parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return PARSER.parseDelimitedFrom(input); + } + public static org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTracker parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return PARSER.parseDelimitedFrom(input, extensionRegistry); + } + public static org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTracker parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return PARSER.parseFrom(input); + } + public static org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTracker parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return PARSER.parseFrom(input, extensionRegistry); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTracker prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + /** + * Protobuf type {@code CellStatsTracker} + */ + public static final class Builder extends + com.google.protobuf.GeneratedMessage.Builder + implements org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTrackerOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.internal_static_CellStatsTracker_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.internal_static_CellStatsTracker_fieldAccessorTable + .ensureFieldAccessorsInitialized( + org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTracker.class, org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTracker.Builder.class); + } + + // Construct using org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTracker.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { + } + } + private static Builder create() { + return new Builder(); + } + + public Builder clear() { + super.clear(); + rowCount_ = 0L; + bitField0_ = (bitField0_ & ~0x00000001); + colCount_ = 0L; + bitField0_ = (bitField0_ & ~0x00000002); + byteCount_ = 0L; + bitField0_ = (bitField0_ & ~0x00000004); + maxColSize_ = 0L; + bitField0_ = (bitField0_ & ~0x00000008); + maxRowSize_ = 0L; + bitField0_ = (bitField0_ & ~0x00000010); + preColSizeVar_ = 0D; + bitField0_ = (bitField0_ & ~0x00000020); + preRowSizeVar_ = 0D; + bitField0_ = (bitField0_ & ~0x00000040); + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.internal_static_CellStatsTracker_descriptor; + } + + public org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTracker getDefaultInstanceForType() { + return org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTracker.getDefaultInstance(); + } + + public org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTracker build() { + org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTracker result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTracker buildPartial() { + org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTracker result = new org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTracker(this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.rowCount_ = rowCount_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.colCount_ = colCount_; + if (((from_bitField0_ & 0x00000004) == 0x00000004)) { + to_bitField0_ |= 0x00000004; + } + result.byteCount_ = byteCount_; + if (((from_bitField0_ & 0x00000008) == 0x00000008)) { + to_bitField0_ |= 0x00000008; + } + result.maxColSize_ = maxColSize_; + if (((from_bitField0_ & 0x00000010) == 0x00000010)) { + to_bitField0_ |= 0x00000010; + } + result.maxRowSize_ = maxRowSize_; + if (((from_bitField0_ & 0x00000020) == 0x00000020)) { + to_bitField0_ |= 0x00000020; + } + result.preColSizeVar_ = preColSizeVar_; + if (((from_bitField0_ & 0x00000040) == 0x00000040)) { + to_bitField0_ |= 0x00000040; + } + result.preRowSizeVar_ = preRowSizeVar_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTracker) { + return mergeFrom((org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTracker)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTracker other) { + if (other == org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTracker.getDefaultInstance()) return this; + if (other.hasRowCount()) { + setRowCount(other.getRowCount()); + } + if (other.hasColCount()) { + setColCount(other.getColCount()); + } + if (other.hasByteCount()) { + setByteCount(other.getByteCount()); + } + if (other.hasMaxColSize()) { + setMaxColSize(other.getMaxColSize()); + } + if (other.hasMaxRowSize()) { + setMaxRowSize(other.getMaxRowSize()); + } + if (other.hasPreColSizeVar()) { + setPreColSizeVar(other.getPreColSizeVar()); + } + if (other.hasPreRowSizeVar()) { + setPreRowSizeVar(other.getPreRowSizeVar()); + } + this.mergeUnknownFields(other.getUnknownFields()); + return this; + } + + public final boolean isInitialized() { + if (!hasRowCount()) { + + return false; + } + if (!hasColCount()) { + + return false; + } + if (!hasByteCount()) { + + return false; + } + if (!hasMaxColSize()) { + + return false; + } + if (!hasMaxRowSize()) { + + return false; + } + if (!hasPreColSizeVar()) { + + return false; + } + if (!hasPreRowSizeVar()) { + + return false; + } + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTracker parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTracker) e.getUnfinishedMessage(); + throw e; + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + private int bitField0_; + + // required int64 row_count = 1; + private long rowCount_ ; + /** + * required int64 row_count = 1; + */ + public boolean hasRowCount() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + /** + * required int64 row_count = 1; + */ + public long getRowCount() { + return rowCount_; + } + /** + * required int64 row_count = 1; + */ + public Builder setRowCount(long value) { + bitField0_ |= 0x00000001; + rowCount_ = value; + onChanged(); + return this; + } + /** + * required int64 row_count = 1; + */ + public Builder clearRowCount() { + bitField0_ = (bitField0_ & ~0x00000001); + rowCount_ = 0L; + onChanged(); + return this; + } + + // required int64 col_count = 2; + private long colCount_ ; + /** + * required int64 col_count = 2; + */ + public boolean hasColCount() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + /** + * required int64 col_count = 2; + */ + public long getColCount() { + return colCount_; + } + /** + * required int64 col_count = 2; + */ + public Builder setColCount(long value) { + bitField0_ |= 0x00000002; + colCount_ = value; + onChanged(); + return this; + } + /** + * required int64 col_count = 2; + */ + public Builder clearColCount() { + bitField0_ = (bitField0_ & ~0x00000002); + colCount_ = 0L; + onChanged(); + return this; + } + + // required int64 byte_count = 3; + private long byteCount_ ; + /** + * required int64 byte_count = 3; + */ + public boolean hasByteCount() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + /** + * required int64 byte_count = 3; + */ + public long getByteCount() { + return byteCount_; + } + /** + * required int64 byte_count = 3; + */ + public Builder setByteCount(long value) { + bitField0_ |= 0x00000004; + byteCount_ = value; + onChanged(); + return this; + } + /** + * required int64 byte_count = 3; + */ + public Builder clearByteCount() { + bitField0_ = (bitField0_ & ~0x00000004); + byteCount_ = 0L; + onChanged(); + return this; + } + + // required int64 max_col_size = 4; + private long maxColSize_ ; + /** + * required int64 max_col_size = 4; + */ + public boolean hasMaxColSize() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + /** + * required int64 max_col_size = 4; + */ + public long getMaxColSize() { + return maxColSize_; + } + /** + * required int64 max_col_size = 4; + */ + public Builder setMaxColSize(long value) { + bitField0_ |= 0x00000008; + maxColSize_ = value; + onChanged(); + return this; + } + /** + * required int64 max_col_size = 4; + */ + public Builder clearMaxColSize() { + bitField0_ = (bitField0_ & ~0x00000008); + maxColSize_ = 0L; + onChanged(); + return this; + } + + // required int64 max_row_size = 5; + private long maxRowSize_ ; + /** + * required int64 max_row_size = 5; + */ + public boolean hasMaxRowSize() { + return ((bitField0_ & 0x00000010) == 0x00000010); + } + /** + * required int64 max_row_size = 5; + */ + public long getMaxRowSize() { + return maxRowSize_; + } + /** + * required int64 max_row_size = 5; + */ + public Builder setMaxRowSize(long value) { + bitField0_ |= 0x00000010; + maxRowSize_ = value; + onChanged(); + return this; + } + /** + * required int64 max_row_size = 5; + */ + public Builder clearMaxRowSize() { + bitField0_ = (bitField0_ & ~0x00000010); + maxRowSize_ = 0L; + onChanged(); + return this; + } + + // required double pre_col_size_var = 6; + private double preColSizeVar_ ; + /** + * required double pre_col_size_var = 6; + */ + public boolean hasPreColSizeVar() { + return ((bitField0_ & 0x00000020) == 0x00000020); + } + /** + * required double pre_col_size_var = 6; + */ + public double getPreColSizeVar() { + return preColSizeVar_; + } + /** + * required double pre_col_size_var = 6; + */ + public Builder setPreColSizeVar(double value) { + bitField0_ |= 0x00000020; + preColSizeVar_ = value; + onChanged(); + return this; + } + /** + * required double pre_col_size_var = 6; + */ + public Builder clearPreColSizeVar() { + bitField0_ = (bitField0_ & ~0x00000020); + preColSizeVar_ = 0D; + onChanged(); + return this; + } + + // required double pre_row_size_var = 7; + private double preRowSizeVar_ ; + /** + * required double pre_row_size_var = 7; + */ + public boolean hasPreRowSizeVar() { + return ((bitField0_ & 0x00000040) == 0x00000040); + } + /** + * required double pre_row_size_var = 7; + */ + public double getPreRowSizeVar() { + return preRowSizeVar_; + } + /** + * required double pre_row_size_var = 7; + */ + public Builder setPreRowSizeVar(double value) { + bitField0_ |= 0x00000040; + preRowSizeVar_ = value; + onChanged(); + return this; + } + /** + * required double pre_row_size_var = 7; + */ + public Builder clearPreRowSizeVar() { + bitField0_ = (bitField0_ & ~0x00000040); + preRowSizeVar_ = 0D; + onChanged(); + return this; + } + + // @@protoc_insertion_point(builder_scope:CellStatsTracker) + } + + static { + defaultInstance = new CellStatsTracker(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:CellStatsTracker) + } + + private static com.google.protobuf.Descriptors.Descriptor + internal_static_CellStatsTracker_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_CellStatsTracker_fieldAccessorTable; + + public static com.google.protobuf.Descriptors.FileDescriptor + getDescriptor() { + return descriptor; + } + private static com.google.protobuf.Descriptors.FileDescriptor + descriptor; + static { + java.lang.String[] descriptorData = { + "\n\021CellTracker.proto\"\254\001\n\020CellStatsTracker" + + "\022\021\n\trow_count\030\001 \002(\003\022\021\n\tcol_count\030\002 \002(\003\022\022" + + "\n\nbyte_count\030\003 \002(\003\022\024\n\014max_col_size\030\004 \002(\003" + + "\022\024\n\014max_row_size\030\005 \002(\003\022\030\n\020pre_col_size_v" + + "ar\030\006 \002(\001\022\030\n\020pre_row_size_var\030\007 \002(\001B?\n*or" + + "g.apache.hadoop.hbase.protobuf.generated" + + "B\021CellTrackerProtos" + }; + com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = + new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { + public com.google.protobuf.ExtensionRegistry assignDescriptors( + com.google.protobuf.Descriptors.FileDescriptor root) { + descriptor = root; + internal_static_CellStatsTracker_descriptor = + getDescriptor().getMessageTypes().get(0); + internal_static_CellStatsTracker_fieldAccessorTable = new + com.google.protobuf.GeneratedMessage.FieldAccessorTable( + internal_static_CellStatsTracker_descriptor, + new java.lang.String[] { "RowCount", "ColCount", "ByteCount", "MaxColSize", "MaxRowSize", "PreColSizeVar", "PreRowSizeVar", }); + return null; + } + }; + com.google.protobuf.Descriptors.FileDescriptor + .internalBuildGeneratedFileFrom(descriptorData, + new com.google.protobuf.Descriptors.FileDescriptor[] { + }, assigner); + } + + // @@protoc_insertion_point(outer_class_scope) +} diff --git a/hbase-protocol/src/main/protobuf/CellTracker.proto b/hbase-protocol/src/main/protobuf/CellTracker.proto new file mode 100644 index 0000000..2b7ae63 --- /dev/null +++ b/hbase-protocol/src/main/protobuf/CellTracker.proto @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +option java_package = "org.apache.hadoop.hbase.protobuf.generated"; +option java_outer_classname = "CellTrackerProtos"; + + message CellStatsTracker { + required int64 row_count = 1; + required int64 col_count = 2; + required int64 byte_count = 3; + required int64 max_col_size = 4; + required int64 max_row_size = 5; + required double pre_col_size_var = 6; + required double pre_row_size_var = 7; +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java index 8b41401..a3baa43 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java @@ -63,6 +63,7 @@ import org.apache.hadoop.hbase.io.compress.Compression; import org.apache.hadoop.hbase.io.crypto.Cipher; import org.apache.hadoop.hbase.io.crypto.Encryption; import org.apache.hadoop.hbase.io.hfile.CacheConfig; +import org.apache.hadoop.hbase.io.hfile.CellStatsTracker; import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.io.hfile.HFileContext; import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder; @@ -234,11 +235,6 @@ public class HStore implements Store { Math.max(conf.getLong("hbase.hstore.time.to.purge.deletes", 0), 0); LOG.trace("Time to purge deletes set to " + timeToPurgeDeletes + "ms in store " + this); - // Get TTL - long ttl = determineTTLFromFamily(family); - // Why not just pass a HColumnDescriptor in here altogether? Even if have - // to clone it? - scanInfo = new ScanInfo(family, ttl, timeToPurgeDeletes, this.comparator); String className = conf.get(MEMSTORE_CLASS_NAME, DefaultMemStore.class.getName()); this.memstore = ReflectionUtils.instantiateWithCustomCtor(className, new Class[] { Configuration.class, KeyValue.KVComparator.class }, new Object[] { conf, this.comparator }); @@ -267,6 +263,25 @@ public class HStore implements Store { this.storeEngine = StoreEngine.create(this, this.conf, this.comparator); this.storeEngine.getStoreFileManager().loadFiles(loadStoreFiles()); + CellStatsTracker combined = null; + for (StoreFile sfile : getStorefiles()) { + byte[] pb = sfile.getReader().loadFileInfo().get(StoreFile.CELLSTATS_KEY); + if (pb != null) { + CellStatsTracker cst = ProtobufUtil.ToCellStatsTracker(org.apache.hadoop.hbase.protobuf.generated.CellTrackerProtos.CellStatsTracker.parseFrom(pb)); + if (combined == null) { + combined = cst; + } else { + combined.combine(cst); + } + } + } + + // Get TTL + long ttl = determineTTLFromFamily(family); + // Why not just pass a HColumnDescriptor in here altogether? Even if have + // to clone it? + scanInfo = new ScanInfo(family, ttl, timeToPurgeDeletes, this.comparator, combined); + // Initialize checksum type from name. The names are CRC32, CRC32C, etc. this.checksumType = getChecksumType(conf); // initilize bytes per checksum diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanInfo.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanInfo.java index a8b314e..6088006 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanInfo.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanInfo.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.regionserver; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.io.hfile.CellStatsTracker; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.KeepDeletedCells; import org.apache.hadoop.hbase.KeyValue.KVComparator; @@ -30,13 +31,14 @@ import org.apache.hadoop.hbase.util.ClassSize; */ @InterfaceAudience.Private public class ScanInfo { - private byte[] family; - private int minVersions; - private int maxVersions; - private long ttl; - private KeepDeletedCells keepDeletedCells; - private long timeToPurgeDeletes; - private KVComparator comparator; + private final byte[] family; + private final int minVersions; + private final int maxVersions; + private final long ttl; + private final KeepDeletedCells keepDeletedCells; + private final long timeToPurgeDeletes; + private final KVComparator comparator; + private final CellStatsTracker statsTracker; public static final long FIXED_OVERHEAD = ClassSize.align(ClassSize.OBJECT + (2 * ClassSize.REFERENCE) + (2 * Bytes.SIZEOF_INT) @@ -48,11 +50,12 @@ public class ScanInfo { * @param timeToPurgeDeletes duration in ms after which a delete marker can * be purged during a major compaction. * @param comparator The store's comparator + * @param statsTracker Stats for store */ public ScanInfo(final HColumnDescriptor family, final long ttl, final long timeToPurgeDeletes, - final KVComparator comparator) { + final KVComparator comparator, CellStatsTracker statsTracker) { this(family.getName(), family.getMinVersions(), family.getMaxVersions(), ttl, family - .getKeepDeletedCells(), timeToPurgeDeletes, comparator); + .getKeepDeletedCells(), timeToPurgeDeletes, comparator, statsTracker); } /** @@ -60,14 +63,15 @@ public class ScanInfo { * @param minVersions Store's MIN_VERSIONS setting * @param maxVersions Store's VERSIONS setting * @param ttl Store's TTL (in ms) + * @param keepDeletedCells Store's keepDeletedCells setting * @param timeToPurgeDeletes duration in ms after which a delete marker can * be purged during a major compaction. - * @param keepDeletedCells Store's keepDeletedCells setting * @param comparator The store's comparator + * @param statsTracker TODO */ public ScanInfo(final byte[] family, final int minVersions, final int maxVersions, final long ttl, final KeepDeletedCells keepDeletedCells, final long timeToPurgeDeletes, - final KVComparator comparator) { + final KVComparator comparator, CellStatsTracker statsTracker) { this.family = family; this.minVersions = minVersions; this.maxVersions = maxVersions; @@ -75,6 +79,7 @@ public class ScanInfo { this.keepDeletedCells = keepDeletedCells; this.timeToPurgeDeletes = timeToPurgeDeletes; this.comparator = comparator; + this.statsTracker = statsTracker; } public byte[] getFamily() { @@ -104,4 +109,8 @@ public class ScanInfo { public KVComparator getComparator() { return comparator; } + + public CellStatsTracker getStatsTracker() { + return statsTracker; + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java index 3e139a5..f99e244 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java @@ -48,10 +48,12 @@ import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper; import org.apache.hadoop.hbase.io.hfile.BlockType; import org.apache.hadoop.hbase.io.hfile.CacheConfig; +import org.apache.hadoop.hbase.io.hfile.CellStatsTracker; import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.io.hfile.HFileContext; import org.apache.hadoop.hbase.io.hfile.HFileScanner; import org.apache.hadoop.hbase.io.hfile.HFileWriterV2; +import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.regionserver.compactions.Compactor; import org.apache.hadoop.hbase.util.BloomFilter; import org.apache.hadoop.hbase.util.BloomFilterFactory; @@ -109,6 +111,9 @@ public class StoreFile { /** Key for Timerange information in metadata*/ public static final byte[] TIMERANGE_KEY = Bytes.toBytes("TIMERANGE"); + /** Key for Cell Stats information in metadata*/ + public static final byte[] CELLSTATS_KEY = Bytes.toBytes("CELLSTATS"); + /** Key for timestamp of earliest-put in metadata*/ public static final byte[] EARLIEST_PUT_TS = Bytes.toBytes("EARLIEST_PUT_TS"); @@ -708,6 +713,7 @@ public class StoreFile { protected int bytesPerChecksum; TimeRangeTracker timeRangeTracker = new TimeRangeTracker(); + CellStatsTracker cellStatsTracker = new CellStatsTracker(); /* isTimeRangeTrackerSet keeps track if the timeRange has already been set * When flushing a memstore, we set TimeRange and use this variable to * indicate that it doesn't need to be calculated again while @@ -795,6 +801,7 @@ public class StoreFile { public void appendTrackedTimestampsToMetadata() throws IOException { appendFileInfo(TIMERANGE_KEY,WritableUtils.toByteArray(timeRangeTracker)); appendFileInfo(EARLIEST_PUT_TS, Bytes.toBytes(earliestPutTs)); + appendFileInfo(CELLSTATS_KEY, ProtobufUtil.toCellStatsTracker(cellStatsTracker).toByteArray()); } /** @@ -820,6 +827,7 @@ public class StoreFile { if (!isTimeRangeTrackerSet) { timeRangeTracker.includeTimestamp(cell); } + cellStatsTracker.track(cell); } private void appendGeneralBloomfilter(final Cell cell) throws IOException { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/NoOpScanPolicyObserver.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/NoOpScanPolicyObserver.java index 36493cd..42989d2 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/NoOpScanPolicyObserver.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/NoOpScanPolicyObserver.java @@ -46,7 +46,7 @@ public class NoOpScanPolicyObserver extends BaseRegionObserver { Store store, KeyValueScanner memstoreScanner, InternalScanner s) throws IOException { ScanInfo oldSI = store.getScanInfo(); ScanInfo scanInfo = new ScanInfo(store.getFamily(), oldSI.getTtl(), - oldSI.getTimeToPurgeDeletes(), oldSI.getComparator()); + oldSI.getTimeToPurgeDeletes(), oldSI.getComparator(), null); Scan scan = new Scan(); scan.setMaxVersions(oldSI.getMaxVersions()); return new StoreScanner(store, scanInfo, scan, Collections.singletonList(memstoreScanner), @@ -63,7 +63,7 @@ public class NoOpScanPolicyObserver extends BaseRegionObserver { // this demonstrates how to override the scanners default behavior ScanInfo oldSI = store.getScanInfo(); ScanInfo scanInfo = new ScanInfo(store.getFamily(), oldSI.getTtl(), - oldSI.getTimeToPurgeDeletes(), oldSI.getComparator()); + oldSI.getTimeToPurgeDeletes(), oldSI.getComparator(), null); Scan scan = new Scan(); scan.setMaxVersions(oldSI.getMaxVersions()); return new StoreScanner(store, scanInfo, scan, scanners, scanType, diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java index 4900ef4..a982c47 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java @@ -193,7 +193,7 @@ public class TestCompaction { ScanInfo old = store.getScanInfo(); ScanInfo si = new ScanInfo(old.getFamily(), old.getMinVersions(), old.getMaxVersions(), ttl, - old.getKeepDeletedCells(), 0, old.getComparator()); + old.getKeepDeletedCells(), 0, old.getComparator(), null); store.setScanInfo(si); } Thread.sleep(ttl); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultCompactSelection.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultCompactSelection.java index a78e1d5..d027caa 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultCompactSelection.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultCompactSelection.java @@ -312,7 +312,7 @@ public class TestDefaultCompactSelection extends TestCase { ScanInfo newScanInfo = new ScanInfo(oldScanInfo.getFamily(), oldScanInfo.getMinVersions(), oldScanInfo.getMaxVersions(), 600, oldScanInfo.getKeepDeletedCells(), oldScanInfo.getTimeToPurgeDeletes(), - oldScanInfo.getComparator()); + oldScanInfo.getComparator(), null); store.setScanInfo(newScanInfo); // Do not compact empty store file List candidates = sfCreate(0); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java index d6d82df..0716b0c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java @@ -96,7 +96,7 @@ public class TestDefaultMemStore extends TestCase { List result = new ArrayList(); ScanInfo scanInfo = new ScanInfo(null, 0, 1, HConstants.LATEST_TIMESTAMP, KeepDeletedCells.FALSE, 0, - this.memstore.comparator); + this.memstore.comparator, null); ScanType scanType = ScanType.USER_SCAN; StoreScanner s = new StoreScanner(scan, scanInfo, scanType, null, memstorescanners); int count = 0; @@ -516,7 +516,7 @@ public class TestDefaultMemStore extends TestCase { //starting from each row, validate results should contain the starting row for (int startRowId = 0; startRowId < ROW_COUNT; startRowId++) { ScanInfo scanInfo = new ScanInfo(FAMILY, 0, 1, Integer.MAX_VALUE, KeepDeletedCells.FALSE, - 0, this.memstore.comparator); + 0, this.memstore.comparator, null); ScanType scanType = ScanType.USER_SCAN; InternalScanner scanner = new StoreScanner(new Scan( Bytes.toBytes(startRowId)), scanInfo, scanType, null, diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java index 0469f0e..48fa52b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java @@ -290,7 +290,7 @@ public class TestMajorCompaction { ScanInfo old = store.getScanInfo(); ScanInfo si = new ScanInfo(old.getFamily(), old.getMinVersions(), old.getMaxVersions(), ttl, - old.getKeepDeletedCells(), 0, old.getComparator()); + old.getKeepDeletedCells(), 0, old.getComparator(), null); store.setScanInfo(si); } Thread.sleep(1000); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java index 97c66e4..af3f116 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java @@ -98,7 +98,7 @@ public class TestQueryMatcher extends HBaseTestCase { private void _testMatch_ExplicitColumns(Scan scan, List expected) throws IOException { // 2,4,5 ScanQueryMatcher qm = new ScanQueryMatcher(scan, new ScanInfo(fam2, - 0, 1, ttl, KeepDeletedCells.FALSE, 0, rowComparator), get.getFamilyMap().get(fam2), + 0, 1, ttl, KeepDeletedCells.FALSE, 0, rowComparator, null), get.getFamilyMap().get(fam2), EnvironmentEdgeManager.currentTime() - ttl); List memstore = new ArrayList(); @@ -183,7 +183,7 @@ public class TestQueryMatcher extends HBaseTestCase { expected.add(ScanQueryMatcher.MatchCode.DONE); ScanQueryMatcher qm = new ScanQueryMatcher(scan, new ScanInfo(fam2, - 0, 1, ttl, KeepDeletedCells.FALSE, 0, rowComparator), null, + 0, 1, ttl, KeepDeletedCells.FALSE, 0, rowComparator, null), null, EnvironmentEdgeManager.currentTime() - ttl); List memstore = new ArrayList(); @@ -239,7 +239,7 @@ public class TestQueryMatcher extends HBaseTestCase { long now = EnvironmentEdgeManager.currentTime(); ScanQueryMatcher qm = new ScanQueryMatcher(scan, new ScanInfo(fam2, 0, 1, testTTL, KeepDeletedCells.FALSE, 0, - rowComparator), get.getFamilyMap().get(fam2), now - testTTL); + rowComparator, null), get.getFamilyMap().get(fam2), now - testTTL); KeyValue [] kvs = new KeyValue[] { new KeyValue(row1, fam2, col1, now-100, data), @@ -293,7 +293,7 @@ public class TestQueryMatcher extends HBaseTestCase { long now = EnvironmentEdgeManager.currentTime(); ScanQueryMatcher qm = new ScanQueryMatcher(scan, new ScanInfo(fam2, - 0, 1, testTTL, KeepDeletedCells.FALSE, 0, rowComparator), null, + 0, 1, testTTL, KeepDeletedCells.FALSE, 0, rowComparator, null), null, now - testTTL); KeyValue [] kvs = new KeyValue[] { @@ -349,7 +349,7 @@ public class TestQueryMatcher extends HBaseTestCase { byte[] from, byte[] to, byte[][] rows, MatchCode... expected) throws IOException { long now = EnvironmentEdgeManager.currentTime(); // Set time to purge deletes to negative value to avoid it ever happening. - ScanInfo scanInfo = new ScanInfo(fam2, 0, 1, ttl, KeepDeletedCells.FALSE, -1L, rowComparator); + ScanInfo scanInfo = new ScanInfo(fam2, 0, 1, ttl, KeepDeletedCells.FALSE, -1L, rowComparator, null); NavigableSet cols = get.getFamilyMap().get(fam2); ScanQueryMatcher qm = new ScanQueryMatcher(scan, scanInfo, cols, Long.MAX_VALUE, diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestReversibleScanners.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestReversibleScanners.java index 5a95df11..0c42ac1 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestReversibleScanners.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestReversibleScanners.java @@ -253,7 +253,7 @@ public class TestReversibleScanners { ScanType scanType = ScanType.USER_SCAN; ScanInfo scanInfo = new ScanInfo(FAMILYNAME, 0, Integer.MAX_VALUE, - Long.MAX_VALUE, KeepDeletedCells.FALSE, 0, KeyValue.COMPARATOR); + Long.MAX_VALUE, KeepDeletedCells.FALSE, 0, KeyValue.COMPARATOR, null); // Case 1.Test a full reversed scan Scan scan = new Scan(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java index bf9fed6..1782482 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java @@ -49,7 +49,7 @@ public class TestStoreScanner extends TestCase { private static final String CF_STR = "cf"; final byte [] CF = Bytes.toBytes(CF_STR); private ScanInfo scanInfo = new ScanInfo(CF, 0, Integer.MAX_VALUE, - Long.MAX_VALUE, KeepDeletedCells.FALSE, 0, KeyValue.COMPARATOR); + Long.MAX_VALUE, KeepDeletedCells.FALSE, 0, KeyValue.COMPARATOR, null); private ScanType scanType = ScanType.USER_SCAN; public void setUp() throws Exception { @@ -418,7 +418,7 @@ public class TestStoreScanner extends TestCase { Scan scan = new Scan(); scan.setMaxVersions(1); ScanInfo scanInfo = new ScanInfo(CF, 0, 1, 500, KeepDeletedCells.FALSE, 0, - KeyValue.COMPARATOR); + KeyValue.COMPARATOR, null); ScanType scanType = ScanType.USER_SCAN; StoreScanner scanner = new StoreScanner(scan, scanInfo, scanType, @@ -489,7 +489,7 @@ public class TestStoreScanner extends TestCase { scan.setMaxVersions(1); // scanner with ttl equal to 500 ScanInfo scanInfo = new ScanInfo(CF, 0, 1, 500, KeepDeletedCells.FALSE, 0, - KeyValue.COMPARATOR); + KeyValue.COMPARATOR, null); ScanType scanType = ScanType.USER_SCAN; StoreScanner scanner = new StoreScanner(scan, scanInfo, scanType, null, scanners); @@ -553,7 +553,7 @@ public class TestStoreScanner extends TestCase { 2 /* maxVersions */, 500 /* ttl */, KeepDeletedCells.FALSE /* keepDeletedCells */, 200, /* timeToPurgeDeletes */ - KeyValue.COMPARATOR); + KeyValue.COMPARATOR, null); StoreScanner scanner = new StoreScanner(scan, scanInfo, ScanType.COMPACT_DROP_DELETES, null, scanners, diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStripeCompactor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStripeCompactor.java index ed8b819..4effdbb 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStripeCompactor.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStripeCompactor.java @@ -176,7 +176,7 @@ public class TestStripeCompactor { // Create store mock that is satisfactory for compactor. HColumnDescriptor col = new HColumnDescriptor(NAME_OF_THINGS); - ScanInfo si = new ScanInfo(col, Long.MAX_VALUE, 0, new KVComparator()); + ScanInfo si = new ScanInfo(col, Long.MAX_VALUE, 0, new KVComparator(), null); Store store = mock(Store.class); when(store.getFamily()).thenReturn(col); when(store.getScanInfo()).thenReturn(si); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java index 034d6bc..9ba16ed 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java @@ -252,7 +252,7 @@ public class TestCoprocessorScanPolicy { ScanInfo scanInfo = new ScanInfo(family.getName(), family.getMinVersions(), newVersions == null ? family.getMaxVersions() : newVersions, newTtl == null ? oldSI.getTtl() : newTtl, family.getKeepDeletedCells(), - oldSI.getTimeToPurgeDeletes(), oldSI.getComparator()); + oldSI.getTimeToPurgeDeletes(), oldSI.getComparator(), null); Scan scan = new Scan(); scan.setMaxVersions(newVersions == null ? oldSI.getMaxVersions() : newVersions); return new StoreScanner(store, scanInfo, scan, Collections.singletonList(memstoreScanner), @@ -272,7 +272,7 @@ public class TestCoprocessorScanPolicy { ScanInfo scanInfo = new ScanInfo(family.getName(), family.getMinVersions(), newVersions == null ? family.getMaxVersions() : newVersions, newTtl == null ? oldSI.getTtl() : newTtl, family.getKeepDeletedCells(), - oldSI.getTimeToPurgeDeletes(), oldSI.getComparator()); + oldSI.getTimeToPurgeDeletes(), oldSI.getComparator(), null); Scan scan = new Scan(); scan.setMaxVersions(newVersions == null ? oldSI.getMaxVersions() : newVersions); return new StoreScanner(store, scanInfo, scan, scanners, scanType, @@ -292,7 +292,7 @@ public class TestCoprocessorScanPolicy { ScanInfo scanInfo = new ScanInfo(family.getName(), family.getMinVersions(), newVersions == null ? family.getMaxVersions() : newVersions, newTtl == null ? oldSI.getTtl() : newTtl, family.getKeepDeletedCells(), - oldSI.getTimeToPurgeDeletes(), oldSI.getComparator()); + oldSI.getTimeToPurgeDeletes(), oldSI.getComparator(), null); return new StoreScanner(store, scanInfo, scan, targetCols, ((HStore) store).getHRegion().getReadpoint(IsolationLevel.READ_COMMITTED)); } else {