From 73a4fe6f320b209da189bdffab06ba247d7f3aab Mon Sep 17 00:00:00 2001 From: Elliott Clark Date: Mon, 17 Dec 2012 15:31:01 -0800 Subject: [PATCH] Move ScanMetrics from writable to Protobuf. --- .../hbase/protobuf/generated/HBaseProtos.java | 504 +++++++++++++++- .../hbase/protobuf/generated/MapReduceProtos.java | 635 ++++++++++++++++++++ hbase-protocol/src/main/protobuf/MapReduce.proto | 32 + hbase-protocol/src/main/protobuf/hbase.proto | 5 + .../apache/hadoop/hbase/client/ClientScanner.java | 10 +- .../hadoop/hbase/client/ScannerCallable.java | 10 +- .../hadoop/hbase/client/metrics/ScanMetrics.java | 141 ++--- .../hbase/mapreduce/TableRecordReaderImpl.java | 17 +- .../apache/hadoop/hbase/protobuf/ProtobufUtil.java | 34 ++ .../hadoop/hbase/client/TestFromClientSide.java | 13 +- 10 files changed, 1281 insertions(+), 120 deletions(-) create mode 100644 hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MapReduceProtos.java create mode 100644 hbase-protocol/src/main/protobuf/MapReduce.proto diff --git hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/HBaseProtos.java hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/HBaseProtos.java index e8de65d..db740bc 100644 --- hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/HBaseProtos.java +++ hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/HBaseProtos.java @@ -10829,6 +10829,480 @@ public final class HBaseProtos { // @@protoc_insertion_point(class_scope:BytesBytesPair) } + public interface NameInt64PairOrBuilder + extends com.google.protobuf.MessageOrBuilder { + + // optional string name = 1; + boolean hasName(); + String getName(); + + // optional int64 value = 2; + boolean hasValue(); + long getValue(); + } + public static final class NameInt64Pair extends + com.google.protobuf.GeneratedMessage + implements NameInt64PairOrBuilder { + // Use NameInt64Pair.newBuilder() to construct. + private NameInt64Pair(Builder builder) { + super(builder); + } + private NameInt64Pair(boolean noInit) {} + + private static final NameInt64Pair defaultInstance; + public static NameInt64Pair getDefaultInstance() { + return defaultInstance; + } + + public NameInt64Pair getDefaultInstanceForType() { + return defaultInstance; + } + + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.internal_static_NameInt64Pair_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.internal_static_NameInt64Pair_fieldAccessorTable; + } + + private int bitField0_; + // optional string name = 1; + public static final int NAME_FIELD_NUMBER = 1; + private java.lang.Object name_; + public boolean hasName() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public String getName() { + java.lang.Object ref = name_; + if (ref instanceof String) { + return (String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + String s = bs.toStringUtf8(); + if (com.google.protobuf.Internal.isValidUtf8(bs)) { + name_ = s; + } + return s; + } + } + private com.google.protobuf.ByteString getNameBytes() { + java.lang.Object ref = name_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8((String) ref); + name_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + // optional int64 value = 2; + public static final int VALUE_FIELD_NUMBER = 2; + private long value_; + public boolean hasValue() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public long getValue() { + return value_; + } + + private void initFields() { + name_ = ""; + value_ = 0L; + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeBytes(1, getNameBytes()); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + output.writeInt64(2, value_); + } + getUnknownFields().writeTo(output); + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(1, getNameBytes()); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.CodedOutputStream + .computeInt64Size(2, value_); + } + size += getUnknownFields().getSerializedSize(); + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64Pair)) { + return super.equals(obj); + } + org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64Pair other = (org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64Pair) obj; + + boolean result = true; + result = result && (hasName() == other.hasName()); + if (hasName()) { + result = result && getName() + .equals(other.getName()); + } + result = result && (hasValue() == other.hasValue()); + if (hasValue()) { + result = result && (getValue() + == other.getValue()); + } + result = result && + getUnknownFields().equals(other.getUnknownFields()); + return result; + } + + @java.lang.Override + public int hashCode() { + int hash = 41; + hash = (19 * hash) + getDescriptorForType().hashCode(); + if (hasName()) { + hash = (37 * hash) + NAME_FIELD_NUMBER; + hash = (53 * hash) + getName().hashCode(); + } + if (hasValue()) { + hash = (37 * hash) + VALUE_FIELD_NUMBER; + hash = (53 * hash) + hashLong(getValue()); + } + hash = (29 * hash) + getUnknownFields().hashCode(); + return hash; + } + + public static org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64Pair parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64Pair parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64Pair parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64Pair parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64Pair parseFrom(java.io.InputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64Pair parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64Pair parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64Pair parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input, extensionRegistry)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64Pair parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64Pair parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64Pair prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + public static final class Builder extends + com.google.protobuf.GeneratedMessage.Builder + implements org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64PairOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.internal_static_NameInt64Pair_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.internal_static_NameInt64Pair_fieldAccessorTable; + } + + // Construct using org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64Pair.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { + } + } + private static Builder create() { + return new Builder(); + } + + public Builder clear() { + super.clear(); + name_ = ""; + bitField0_ = (bitField0_ & ~0x00000001); + value_ = 0L; + bitField0_ = (bitField0_ & ~0x00000002); + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64Pair.getDescriptor(); + } + + public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64Pair getDefaultInstanceForType() { + return org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64Pair.getDefaultInstance(); + } + + public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64Pair build() { + org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64Pair result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + private org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64Pair buildParsed() + throws com.google.protobuf.InvalidProtocolBufferException { + org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64Pair result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException( + result).asInvalidProtocolBufferException(); + } + return result; + } + + public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64Pair buildPartial() { + org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64Pair result = new org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64Pair(this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.name_ = name_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.value_ = value_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64Pair) { + return mergeFrom((org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64Pair)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64Pair other) { + if (other == org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64Pair.getDefaultInstance()) return this; + if (other.hasName()) { + setName(other.getName()); + } + if (other.hasValue()) { + setValue(other.getValue()); + } + this.mergeUnknownFields(other.getUnknownFields()); + return this; + } + + public final boolean isInitialized() { + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder( + this.getUnknownFields()); + while (true) { + int tag = input.readTag(); + switch (tag) { + case 0: + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + default: { + if (!parseUnknownField(input, unknownFields, + extensionRegistry, tag)) { + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + } + break; + } + case 10: { + bitField0_ |= 0x00000001; + name_ = input.readBytes(); + break; + } + case 16: { + bitField0_ |= 0x00000002; + value_ = input.readInt64(); + break; + } + } + } + } + + private int bitField0_; + + // optional string name = 1; + private java.lang.Object name_ = ""; + public boolean hasName() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public String getName() { + java.lang.Object ref = name_; + if (!(ref instanceof String)) { + String s = ((com.google.protobuf.ByteString) ref).toStringUtf8(); + name_ = s; + return s; + } else { + return (String) ref; + } + } + public Builder setName(String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + name_ = value; + onChanged(); + return this; + } + public Builder clearName() { + bitField0_ = (bitField0_ & ~0x00000001); + name_ = getDefaultInstance().getName(); + onChanged(); + return this; + } + void setName(com.google.protobuf.ByteString value) { + bitField0_ |= 0x00000001; + name_ = value; + onChanged(); + } + + // optional int64 value = 2; + private long value_ ; + public boolean hasValue() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public long getValue() { + return value_; + } + public Builder setValue(long value) { + bitField0_ |= 0x00000002; + value_ = value; + onChanged(); + return this; + } + public Builder clearValue() { + bitField0_ = (bitField0_ & ~0x00000002); + value_ = 0L; + onChanged(); + return this; + } + + // @@protoc_insertion_point(builder_scope:NameInt64Pair) + } + + static { + defaultInstance = new NameInt64Pair(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:NameInt64Pair) + } + private static com.google.protobuf.Descriptors.Descriptor internal_static_TableSchema_descriptor; private static @@ -10909,6 +11383,11 @@ public final class HBaseProtos { private static com.google.protobuf.GeneratedMessage.FieldAccessorTable internal_static_BytesBytesPair_fieldAccessorTable; + private static com.google.protobuf.Descriptors.Descriptor + internal_static_NameInt64Pair_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_NameInt64Pair_fieldAccessorTable; public static com.google.protobuf.Descriptors.FileDescriptor getDescriptor() { @@ -10960,14 +11439,15 @@ public final class HBaseProtos { " \002(\t\"-\n\016NameStringPair\022\014\n\004name\030\001 \002(\t\022\r\n\005", "value\030\002 \002(\t\",\n\rNameBytesPair\022\014\n\004name\030\001 \002" + "(\t\022\r\n\005value\030\002 \001(\014\"/\n\016BytesBytesPair\022\r\n\005f" + - "irst\030\001 \002(\014\022\016\n\006second\030\002 \002(\014*r\n\013CompareTyp" + - "e\022\010\n\004LESS\020\000\022\021\n\rLESS_OR_EQUAL\020\001\022\t\n\005EQUAL\020" + - "\002\022\r\n\tNOT_EQUAL\020\003\022\024\n\020GREATER_OR_EQUAL\020\004\022\013" + - "\n\007GREATER\020\005\022\t\n\005NO_OP\020\006*_\n\007KeyType\022\013\n\007MIN" + - "IMUM\020\000\022\007\n\003PUT\020\004\022\n\n\006DELETE\020\010\022\021\n\rDELETE_CO" + - "LUMN\020\014\022\021\n\rDELETE_FAMILY\020\016\022\014\n\007MAXIMUM\020\377\001B" + - ">\n*org.apache.hadoop.hbase.protobuf.gene" + - "ratedB\013HBaseProtosH\001\240\001\001" + "irst\030\001 \002(\014\022\016\n\006second\030\002 \002(\014\",\n\rNameInt64P" + + "air\022\014\n\004name\030\001 \001(\t\022\r\n\005value\030\002 \001(\003*r\n\013Comp" + + "areType\022\010\n\004LESS\020\000\022\021\n\rLESS_OR_EQUAL\020\001\022\t\n\005" + + "EQUAL\020\002\022\r\n\tNOT_EQUAL\020\003\022\024\n\020GREATER_OR_EQU" + + "AL\020\004\022\013\n\007GREATER\020\005\022\t\n\005NO_OP\020\006*_\n\007KeyType\022" + + "\013\n\007MINIMUM\020\000\022\007\n\003PUT\020\004\022\n\n\006DELETE\020\010\022\021\n\rDEL" + + "ETE_COLUMN\020\014\022\021\n\rDELETE_FAMILY\020\016\022\014\n\007MAXIM" + + "UM\020\377\001B>\n*org.apache.hadoop.hbase.protobu", + "f.generatedB\013HBaseProtosH\001\240\001\001" }; com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { @@ -11102,6 +11582,14 @@ public final class HBaseProtos { new java.lang.String[] { "First", "Second", }, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.BytesBytesPair.class, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.BytesBytesPair.Builder.class); + internal_static_NameInt64Pair_descriptor = + getDescriptor().getMessageTypes().get(14); + internal_static_NameInt64Pair_fieldAccessorTable = new + com.google.protobuf.GeneratedMessage.FieldAccessorTable( + internal_static_NameInt64Pair_descriptor, + new java.lang.String[] { "Name", "Value", }, + org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64Pair.class, + org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64Pair.Builder.class); return null; } }; diff --git hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MapReduceProtos.java hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MapReduceProtos.java new file mode 100644 index 0000000..5879e96 --- /dev/null +++ hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MapReduceProtos.java @@ -0,0 +1,635 @@ +// Generated by the protocol buffer compiler. DO NOT EDIT! +// source: MapReduce.proto + +package org.apache.hadoop.hbase.protobuf.generated; + +public final class MapReduceProtos { + private MapReduceProtos() {} + public static void registerAllExtensions( + com.google.protobuf.ExtensionRegistry registry) { + } + public interface ScanMetricsOrBuilder + extends com.google.protobuf.MessageOrBuilder { + + // repeated .NameInt64Pair metrics = 1; + java.util.List + getMetricsList(); + org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64Pair getMetrics(int index); + int getMetricsCount(); + java.util.List + getMetricsOrBuilderList(); + org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64PairOrBuilder getMetricsOrBuilder( + int index); + } + public static final class ScanMetrics extends + com.google.protobuf.GeneratedMessage + implements ScanMetricsOrBuilder { + // Use ScanMetrics.newBuilder() to construct. + private ScanMetrics(Builder builder) { + super(builder); + } + private ScanMetrics(boolean noInit) {} + + private static final ScanMetrics defaultInstance; + public static ScanMetrics getDefaultInstance() { + return defaultInstance; + } + + public ScanMetrics getDefaultInstanceForType() { + return defaultInstance; + } + + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.internal_static_ScanMetrics_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.internal_static_ScanMetrics_fieldAccessorTable; + } + + // repeated .NameInt64Pair metrics = 1; + public static final int METRICS_FIELD_NUMBER = 1; + private java.util.List metrics_; + public java.util.List getMetricsList() { + return metrics_; + } + public java.util.List + getMetricsOrBuilderList() { + return metrics_; + } + public int getMetricsCount() { + return metrics_.size(); + } + public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64Pair getMetrics(int index) { + return metrics_.get(index); + } + public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64PairOrBuilder getMetricsOrBuilder( + int index) { + return metrics_.get(index); + } + + private void initFields() { + metrics_ = java.util.Collections.emptyList(); + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + for (int i = 0; i < metrics_.size(); i++) { + output.writeMessage(1, metrics_.get(i)); + } + getUnknownFields().writeTo(output); + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + for (int i = 0; i < metrics_.size(); i++) { + size += com.google.protobuf.CodedOutputStream + .computeMessageSize(1, metrics_.get(i)); + } + size += getUnknownFields().getSerializedSize(); + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.ScanMetrics)) { + return super.equals(obj); + } + org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.ScanMetrics other = (org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.ScanMetrics) obj; + + boolean result = true; + result = result && getMetricsList() + .equals(other.getMetricsList()); + result = result && + getUnknownFields().equals(other.getUnknownFields()); + return result; + } + + @java.lang.Override + public int hashCode() { + int hash = 41; + hash = (19 * hash) + getDescriptorForType().hashCode(); + if (getMetricsCount() > 0) { + hash = (37 * hash) + METRICS_FIELD_NUMBER; + hash = (53 * hash) + getMetricsList().hashCode(); + } + hash = (29 * hash) + getUnknownFields().hashCode(); + return hash; + } + + public static org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.ScanMetrics parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.ScanMetrics parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.ScanMetrics parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.ScanMetrics parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.ScanMetrics parseFrom(java.io.InputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.ScanMetrics parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.ScanMetrics parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.ScanMetrics parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input, extensionRegistry)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.ScanMetrics parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.ScanMetrics parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.ScanMetrics prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + public static final class Builder extends + com.google.protobuf.GeneratedMessage.Builder + implements org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.ScanMetricsOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.internal_static_ScanMetrics_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.internal_static_ScanMetrics_fieldAccessorTable; + } + + // Construct using org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.ScanMetrics.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { + getMetricsFieldBuilder(); + } + } + private static Builder create() { + return new Builder(); + } + + public Builder clear() { + super.clear(); + if (metricsBuilder_ == null) { + metrics_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000001); + } else { + metricsBuilder_.clear(); + } + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.ScanMetrics.getDescriptor(); + } + + public org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.ScanMetrics getDefaultInstanceForType() { + return org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.ScanMetrics.getDefaultInstance(); + } + + public org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.ScanMetrics build() { + org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.ScanMetrics result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + private org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.ScanMetrics buildParsed() + throws com.google.protobuf.InvalidProtocolBufferException { + org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.ScanMetrics result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException( + result).asInvalidProtocolBufferException(); + } + return result; + } + + public org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.ScanMetrics buildPartial() { + org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.ScanMetrics result = new org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.ScanMetrics(this); + int from_bitField0_ = bitField0_; + if (metricsBuilder_ == null) { + if (((bitField0_ & 0x00000001) == 0x00000001)) { + metrics_ = java.util.Collections.unmodifiableList(metrics_); + bitField0_ = (bitField0_ & ~0x00000001); + } + result.metrics_ = metrics_; + } else { + result.metrics_ = metricsBuilder_.build(); + } + onBuilt(); + return result; + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.ScanMetrics) { + return mergeFrom((org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.ScanMetrics)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.ScanMetrics other) { + if (other == org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.ScanMetrics.getDefaultInstance()) return this; + if (metricsBuilder_ == null) { + if (!other.metrics_.isEmpty()) { + if (metrics_.isEmpty()) { + metrics_ = other.metrics_; + bitField0_ = (bitField0_ & ~0x00000001); + } else { + ensureMetricsIsMutable(); + metrics_.addAll(other.metrics_); + } + onChanged(); + } + } else { + if (!other.metrics_.isEmpty()) { + if (metricsBuilder_.isEmpty()) { + metricsBuilder_.dispose(); + metricsBuilder_ = null; + metrics_ = other.metrics_; + bitField0_ = (bitField0_ & ~0x00000001); + metricsBuilder_ = + com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ? + getMetricsFieldBuilder() : null; + } else { + metricsBuilder_.addAllMessages(other.metrics_); + } + } + } + this.mergeUnknownFields(other.getUnknownFields()); + return this; + } + + public final boolean isInitialized() { + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder( + this.getUnknownFields()); + while (true) { + int tag = input.readTag(); + switch (tag) { + case 0: + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + default: { + if (!parseUnknownField(input, unknownFields, + extensionRegistry, tag)) { + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + } + break; + } + case 10: { + org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64Pair.Builder subBuilder = org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64Pair.newBuilder(); + input.readMessage(subBuilder, extensionRegistry); + addMetrics(subBuilder.buildPartial()); + break; + } + } + } + } + + private int bitField0_; + + // repeated .NameInt64Pair metrics = 1; + private java.util.List metrics_ = + java.util.Collections.emptyList(); + private void ensureMetricsIsMutable() { + if (!((bitField0_ & 0x00000001) == 0x00000001)) { + metrics_ = new java.util.ArrayList(metrics_); + bitField0_ |= 0x00000001; + } + } + + private com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64Pair, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64Pair.Builder, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64PairOrBuilder> metricsBuilder_; + + public java.util.List getMetricsList() { + if (metricsBuilder_ == null) { + return java.util.Collections.unmodifiableList(metrics_); + } else { + return metricsBuilder_.getMessageList(); + } + } + public int getMetricsCount() { + if (metricsBuilder_ == null) { + return metrics_.size(); + } else { + return metricsBuilder_.getCount(); + } + } + public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64Pair getMetrics(int index) { + if (metricsBuilder_ == null) { + return metrics_.get(index); + } else { + return metricsBuilder_.getMessage(index); + } + } + public Builder setMetrics( + int index, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64Pair value) { + if (metricsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureMetricsIsMutable(); + metrics_.set(index, value); + onChanged(); + } else { + metricsBuilder_.setMessage(index, value); + } + return this; + } + public Builder setMetrics( + int index, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64Pair.Builder builderForValue) { + if (metricsBuilder_ == null) { + ensureMetricsIsMutable(); + metrics_.set(index, builderForValue.build()); + onChanged(); + } else { + metricsBuilder_.setMessage(index, builderForValue.build()); + } + return this; + } + public Builder addMetrics(org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64Pair value) { + if (metricsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureMetricsIsMutable(); + metrics_.add(value); + onChanged(); + } else { + metricsBuilder_.addMessage(value); + } + return this; + } + public Builder addMetrics( + int index, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64Pair value) { + if (metricsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureMetricsIsMutable(); + metrics_.add(index, value); + onChanged(); + } else { + metricsBuilder_.addMessage(index, value); + } + return this; + } + public Builder addMetrics( + org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64Pair.Builder builderForValue) { + if (metricsBuilder_ == null) { + ensureMetricsIsMutable(); + metrics_.add(builderForValue.build()); + onChanged(); + } else { + metricsBuilder_.addMessage(builderForValue.build()); + } + return this; + } + public Builder addMetrics( + int index, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64Pair.Builder builderForValue) { + if (metricsBuilder_ == null) { + ensureMetricsIsMutable(); + metrics_.add(index, builderForValue.build()); + onChanged(); + } else { + metricsBuilder_.addMessage(index, builderForValue.build()); + } + return this; + } + public Builder addAllMetrics( + java.lang.Iterable values) { + if (metricsBuilder_ == null) { + ensureMetricsIsMutable(); + super.addAll(values, metrics_); + onChanged(); + } else { + metricsBuilder_.addAllMessages(values); + } + return this; + } + public Builder clearMetrics() { + if (metricsBuilder_ == null) { + metrics_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000001); + onChanged(); + } else { + metricsBuilder_.clear(); + } + return this; + } + public Builder removeMetrics(int index) { + if (metricsBuilder_ == null) { + ensureMetricsIsMutable(); + metrics_.remove(index); + onChanged(); + } else { + metricsBuilder_.remove(index); + } + return this; + } + public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64Pair.Builder getMetricsBuilder( + int index) { + return getMetricsFieldBuilder().getBuilder(index); + } + public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64PairOrBuilder getMetricsOrBuilder( + int index) { + if (metricsBuilder_ == null) { + return metrics_.get(index); } else { + return metricsBuilder_.getMessageOrBuilder(index); + } + } + public java.util.List + getMetricsOrBuilderList() { + if (metricsBuilder_ != null) { + return metricsBuilder_.getMessageOrBuilderList(); + } else { + return java.util.Collections.unmodifiableList(metrics_); + } + } + public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64Pair.Builder addMetricsBuilder() { + return getMetricsFieldBuilder().addBuilder( + org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64Pair.getDefaultInstance()); + } + public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64Pair.Builder addMetricsBuilder( + int index) { + return getMetricsFieldBuilder().addBuilder( + index, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64Pair.getDefaultInstance()); + } + public java.util.List + getMetricsBuilderList() { + return getMetricsFieldBuilder().getBuilderList(); + } + private com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64Pair, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64Pair.Builder, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64PairOrBuilder> + getMetricsFieldBuilder() { + if (metricsBuilder_ == null) { + metricsBuilder_ = new com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64Pair, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64Pair.Builder, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64PairOrBuilder>( + metrics_, + ((bitField0_ & 0x00000001) == 0x00000001), + getParentForChildren(), + isClean()); + metrics_ = null; + } + return metricsBuilder_; + } + + // @@protoc_insertion_point(builder_scope:ScanMetrics) + } + + static { + defaultInstance = new ScanMetrics(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:ScanMetrics) + } + + private static com.google.protobuf.Descriptors.Descriptor + internal_static_ScanMetrics_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_ScanMetrics_fieldAccessorTable; + + public static com.google.protobuf.Descriptors.FileDescriptor + getDescriptor() { + return descriptor; + } + private static com.google.protobuf.Descriptors.FileDescriptor + descriptor; + static { + java.lang.String[] descriptorData = { + "\n\017MapReduce.proto\032\013hbase.proto\".\n\013ScanMe" + + "trics\022\037\n\007metrics\030\001 \003(\0132\016.NameInt64PairBB" + + "\n*org.apache.hadoop.hbase.protobuf.gener" + + "atedB\017MapReduceProtosH\001\240\001\001" + }; + com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = + new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { + public com.google.protobuf.ExtensionRegistry assignDescriptors( + com.google.protobuf.Descriptors.FileDescriptor root) { + descriptor = root; + internal_static_ScanMetrics_descriptor = + getDescriptor().getMessageTypes().get(0); + internal_static_ScanMetrics_fieldAccessorTable = new + com.google.protobuf.GeneratedMessage.FieldAccessorTable( + internal_static_ScanMetrics_descriptor, + new java.lang.String[] { "Metrics", }, + org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.ScanMetrics.class, + org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.ScanMetrics.Builder.class); + return null; + } + }; + com.google.protobuf.Descriptors.FileDescriptor + .internalBuildGeneratedFileFrom(descriptorData, + new com.google.protobuf.Descriptors.FileDescriptor[] { + org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.getDescriptor(), + }, assigner); + } + + // @@protoc_insertion_point(outer_class_scope) +} diff --git hbase-protocol/src/main/protobuf/MapReduce.proto hbase-protocol/src/main/protobuf/MapReduce.proto new file mode 100644 index 0000000..0ac70cd --- /dev/null +++ hbase-protocol/src/main/protobuf/MapReduce.proto @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + //This file includes protocol buffers used in MapReduce only. + + option java_package = "org.apache.hadoop.hbase.protobuf.generated"; + option java_outer_classname = "MapReduceProtos"; + option java_generate_equals_and_hash = true; + option optimize_for = SPEED; + + import "hbase.proto"; + + message ScanMetrics { + + repeated NameInt64Pair metrics = 1; + + } \ No newline at end of file diff --git hbase-protocol/src/main/protobuf/hbase.proto hbase-protocol/src/main/protobuf/hbase.proto index 9d74b17..1ff2a4d 100644 --- hbase-protocol/src/main/protobuf/hbase.proto +++ hbase-protocol/src/main/protobuf/hbase.proto @@ -263,3 +263,8 @@ message BytesBytesPair { required bytes first = 1; required bytes second = 2; } + +message NameInt64Pair { + optional string name = 1; + optional int64 value = 2; +} diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java hbase-server/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java index 18091b5..553346b 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java @@ -34,6 +34,8 @@ import org.apache.hadoop.hbase.NotServingRegionException; import org.apache.hadoop.hbase.OutOfOrderScannerNextException; import org.apache.hadoop.hbase.UnknownScannerException; import org.apache.hadoop.hbase.client.metrics.ScanMetrics; +import org.apache.hadoop.hbase.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos; import org.apache.hadoop.hbase.regionserver.RegionServerStoppedException; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.io.DataOutputBuffer; @@ -215,7 +217,7 @@ public class ClientScanner extends AbstractClientScanner { callable.withRetries(); this.currentRegion = callable.getHRegionInfo(); if (this.scanMetrics != null) { - this.scanMetrics.countOfRegions.inc(); + this.scanMetrics.countOfRegions.incrementAndGet(); } } catch (IOException e) { close(); @@ -249,8 +251,8 @@ public class ClientScanner extends AbstractClientScanner { return; } final DataOutputBuffer d = new DataOutputBuffer(); - scanMetrics.write(d); - scan.setAttribute(Scan.SCAN_ATTRIBUTES_METRICS_DATA, d.getData()); + MapReduceProtos.ScanMetrics pScanMetrics = ProtobufUtil.toScanMetrics(scanMetrics); + scan.setAttribute(Scan.SCAN_ATTRIBUTES_METRICS_DATA, pScanMetrics.toByteArray()); } public Result next() throws IOException { @@ -329,7 +331,7 @@ public class ClientScanner extends AbstractClientScanner { } long currentTime = System.currentTimeMillis(); if (this.scanMetrics != null ) { - this.scanMetrics.sumOfMillisSecBetweenNexts.inc(currentTime-lastNext); + this.scanMetrics.sumOfMillisSecBetweenNexts.addAndGet(currentTime-lastNext); } lastNext = currentTime; if (values != null && values.length > 0) { diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java hbase-server/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java index f207af9..634d774 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java @@ -102,9 +102,9 @@ public class ScannerCallable extends ServerCallable { // HConnectionManager will call instantiateServer with reload==true // if and only if for retries. if (reload && this.scanMetrics != null) { - this.scanMetrics.countOfRPCRetries.inc(); + this.scanMetrics.countOfRPCRetries.incrementAndGet(); if (isRegionServerRemote) { - this.scanMetrics.countOfRemoteRPCRetries.inc(); + this.scanMetrics.countOfRemoteRPCRetries.incrementAndGet(); } } } @@ -197,7 +197,7 @@ public class ScannerCallable extends ServerCallable { // when what we need is to open scanner against new location. // Attach NSRE to signal client that it needs to resetup scanner. if (this.scanMetrics != null) { - this.scanMetrics.countOfNSRE.inc(); + this.scanMetrics.countOfNSRE.incrementAndGet(); } throw new DoNotRetryIOException("Reset scanner", ioe); } else if (ioe instanceof RegionServerStoppedException) { @@ -220,9 +220,9 @@ public class ScannerCallable extends ServerCallable { if (this.scanMetrics == null) { return; } - this.scanMetrics.countOfRPCcalls.inc(); + this.scanMetrics.countOfRPCcalls.incrementAndGet(); if (isRegionServerRemote) { - this.scanMetrics.countOfRemoteRPCcalls.inc(); + this.scanMetrics.countOfRemoteRPCcalls.incrementAndGet(); } } diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/client/metrics/ScanMetrics.java hbase-server/src/main/java/org/apache/hadoop/hbase/client/metrics/ScanMetrics.java index 57f2af2..47b484c 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/client/metrics/ScanMetrics.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/client/metrics/ScanMetrics.java @@ -18,158 +18,125 @@ package org.apache.hadoop.hbase.client.metrics; -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; - +import com.google.common.collect.ImmutableMap; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.io.Writable; -import org.apache.hadoop.metrics.util.MetricsBase; -import org.apache.hadoop.metrics.util.MetricsRegistry; -import org.apache.hadoop.metrics.util.MetricsTimeVaryingLong; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; /** * Provides client-side metrics related to scan operations * The data can be passed to mapreduce framework or other systems. - * Currently metrics framework won't be able to support the scenario - * where multiple scan instances run on the same machine trying to - * update the same metric. We use metrics objects in the class, - * so that it can be easily switched to metrics framework later when it support - * this scenario. + * We use atomic longs so that one thread can increment, + * while another atomically resets to zero after the values are reported + * to hadoop's counters. + * * Some of these metrics are general for any client operation such as put * However, there is no need for this. So they are defined under scan operation * for now. */ @InterfaceAudience.Public @InterfaceStability.Evolving -public class ScanMetrics implements Writable { +public class ScanMetrics { + - private static final byte SCANMETRICS_VERSION = (byte)1; private static final Log LOG = LogFactory.getLog(ScanMetrics.class); - private MetricsRegistry registry = new MetricsRegistry(); + + /** + * Hash to hold the String -> Atomic Long mappings. + */ + private final Map counters = new HashMap(); + + // AtomicLongs to hold the metrics values. These are all updated through ClientScanner and + // ScannerCallable. They are atomic longs so that atomic getAndSet can be used to reset the + // values after progress is passed to hadoop's counters. + /** * number of RPC calls */ - public final MetricsTimeVaryingLong countOfRPCcalls = - new MetricsTimeVaryingLong("RPC_CALLS", registry); + public final AtomicLong countOfRPCcalls = createCounter("RPC_CALLS"); /** * number of remote RPC calls */ - public final MetricsTimeVaryingLong countOfRemoteRPCcalls = - new MetricsTimeVaryingLong("REMOTE_RPC_CALLS", registry); + public final AtomicLong countOfRemoteRPCcalls = createCounter("REMOTE_RPC_CALLS"); /** * sum of milliseconds between sequential next calls */ - public final MetricsTimeVaryingLong sumOfMillisSecBetweenNexts = - new MetricsTimeVaryingLong("MILLIS_BETWEEN_NEXTS", registry); + public final AtomicLong sumOfMillisSecBetweenNexts = createCounter("MILLIS_BETWEEN_NEXTS"); /** * number of NotServingRegionException caught */ - public final MetricsTimeVaryingLong countOfNSRE = - new MetricsTimeVaryingLong("NOT_SERVING_REGION_EXCEPTION", registry); + public final AtomicLong countOfNSRE = createCounter("NOT_SERVING_REGION_EXCEPTION"); /** * number of bytes in Result objects from region servers */ - public final MetricsTimeVaryingLong countOfBytesInResults = - new MetricsTimeVaryingLong("BYTES_IN_RESULTS", registry); + public final AtomicLong countOfBytesInResults = createCounter("BYTES_IN_RESULTS"); /** * number of bytes in Result objects from remote region servers */ - public final MetricsTimeVaryingLong countOfBytesInRemoteResults = - new MetricsTimeVaryingLong("BYTES_IN_REMOTE_RESULTS", registry); + public final AtomicLong countOfBytesInRemoteResults = createCounter("BYTES_IN_REMOTE_RESULTS"); /** * number of regions */ - public final MetricsTimeVaryingLong countOfRegions = - new MetricsTimeVaryingLong("REGIONS_SCANNED", registry); + public final AtomicLong countOfRegions = createCounter("REGIONS_SCANNED"); /** * number of RPC retries */ - public final MetricsTimeVaryingLong countOfRPCRetries = - new MetricsTimeVaryingLong("RPC_RETRIES", registry); + public final AtomicLong countOfRPCRetries = createCounter("RPC_RETRIES"); /** * number of remote RPC retries */ - public final MetricsTimeVaryingLong countOfRemoteRPCRetries = - new MetricsTimeVaryingLong("REMOTE_RPC_RETRIES", registry); + public final AtomicLong countOfRemoteRPCRetries = createCounter("REMOTE_RPC_RETRIES"); /** * constructor */ - public ScanMetrics () { + public ScanMetrics() { } - /** - * serialize all the MetricsTimeVaryingLong - */ - public void write(DataOutput out) throws IOException { - out.writeByte(SCANMETRICS_VERSION); - Collection mbs = registry.getMetricsList(); - - // we only handle MetricsTimeVaryingLong for now. - int metricsCount = 0; - for (MetricsBase mb : mbs) { - if ( mb instanceof MetricsTimeVaryingLong) { - metricsCount++; - } else { - throw new IOException("unsupported metrics type. metrics name: " - + mb.getName() + ", metrics description: " + mb.getDescription()); - } - } - - out.writeInt(metricsCount); - for (MetricsBase mb : mbs) { - out.writeUTF(mb.getName()); - out.writeLong(((MetricsTimeVaryingLong) mb).getCurrentIntervalValue()); - } + private AtomicLong createCounter(String counterName) { + AtomicLong c = new AtomicLong(0); + counters.put(counterName, c); + return c; } - public void readFields(DataInput in) throws IOException { - int version = in.readByte(); - if (version > (int)SCANMETRICS_VERSION) { - throw new IOException("version " + version + " not supported"); - } - - int metricsCount = in.readInt(); - for (int i=0; i mbs = registry.getMetricsList(); - ArrayList mlv = - new ArrayList(); - for (MetricsBase mb : mbs) { - if ( mb instanceof MetricsTimeVaryingLong) { - mlv.add((MetricsTimeVaryingLong) mb); - } + /** + * Get all of the values since the last time this function was called. + * + * Calling this function will reset all AtomicLongs in the instance back to 0. + * + * @return A Map of String -> Long for metrics + */ + public Map getMetricsMap() { + //Create a builder + ImmutableMap.Builder builder = ImmutableMap.builder(); + //For every entry add the value and reset the AtomicLong back to zero + for (Map.Entry e : this.counters.entrySet()) { + builder.put(e.getKey(), e.getValue().getAndSet(0)); } - return mlv.toArray(new MetricsTimeVaryingLong[mlv.size()]); + //Build the immutable map so that people can't mess around with it. + return builder.build(); } } diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java index 7994aa0..641f363 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.mapreduce; import java.io.IOException; import java.lang.reflect.Method; +import java.util.Map; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -32,6 +33,7 @@ import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.ScannerCallable; import org.apache.hadoop.hbase.client.metrics.ScanMetrics; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.io.DataInputBuffer; import org.apache.hadoop.mapreduce.Counter; @@ -262,19 +264,16 @@ public class TableRecordReaderImpl { return; } - DataInputBuffer in = new DataInputBuffer(); - in.reset(serializedMetrics, 0, serializedMetrics.length); - ScanMetrics scanMetrics = new ScanMetrics(); - scanMetrics.readFields(in); - MetricsTimeVaryingLong[] mlvs = - scanMetrics.getMetricsTimeVaryingLongArray(); + ScanMetrics scanMetrics = ProtobufUtil.toScanMetrics(serializedMetrics); try { - for (MetricsTimeVaryingLong mlv : mlvs) { + for (Map.Entry entry:scanMetrics.getMetricsMap().entrySet()) { Counter ct = (Counter)this.getCounter.invoke(context, - HBASE_COUNTER_GROUP_NAME, mlv.getName()); - ct.increment(mlv.getCurrentIntervalValue()); + HBASE_COUNTER_GROUP_NAME, entry.getKey()); + + ct.increment(entry.getValue()); } + ((Counter) this.getCounter.invoke(context, HBASE_COUNTER_GROUP_NAME, "NUM_SCANNER_RESTARTS")).increment(numRestarts); } catch (Exception e) { diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java hbase-server/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java index 1248144..b761860 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java @@ -39,6 +39,7 @@ import java.util.NavigableMap; import java.util.NavigableSet; import java.util.TreeMap; +import com.google.protobuf.InvalidProtocolBufferException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.DeserializationException; import org.apache.hadoop.hbase.DoNotRetryIOException; @@ -67,6 +68,7 @@ import org.apache.hadoop.hbase.client.RowMutations; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.coprocessor.Exec; import org.apache.hadoop.hbase.client.coprocessor.ExecResult; +import org.apache.hadoop.hbase.client.metrics.ScanMetrics; import org.apache.hadoop.hbase.filter.ByteArrayComparable; import org.apache.hadoop.hbase.filter.Filter; import org.apache.hadoop.hbase.io.HbaseObjectWritable; @@ -116,6 +118,7 @@ import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionLoad; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.RegionSpecifierType; +import org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos; import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.CreateTableRequest; import org.apache.hadoop.hbase.protobuf.generated.MasterMonitorProtos.GetTableDescriptorsResponse; import org.apache.hadoop.hbase.regionserver.wal.HLog; @@ -1833,6 +1836,37 @@ public final class ProtobufUtil { } } + public static ScanMetrics toScanMetrics(final byte[] bytes) { + MapReduceProtos.ScanMetrics.Builder builder = MapReduceProtos.ScanMetrics.newBuilder(); + try { + builder.mergeFrom(bytes); + } catch (InvalidProtocolBufferException e) { + //Ignored there are just no key values to add. + } + MapReduceProtos.ScanMetrics pScanMetrics = builder.build(); + ScanMetrics scanMetrics = new ScanMetrics(); + for (HBaseProtos.NameInt64Pair pair : pScanMetrics.getMetricsList()) { + if (pair.hasName() && pair.hasValue()) { + scanMetrics.setCounter(pair.getName(), pair.getValue()); + } + } + return scanMetrics; + } + + public static MapReduceProtos.ScanMetrics toScanMetrics(ScanMetrics scanMetrics) { + MapReduceProtos.ScanMetrics.Builder builder = MapReduceProtos.ScanMetrics.newBuilder(); + Map metrics = scanMetrics.getMetricsMap(); + for (Entry e : metrics.entrySet()) { + HBaseProtos.NameInt64Pair nameInt64Pair = + HBaseProtos.NameInt64Pair.newBuilder() + .setName(e.getKey()) + .setValue(e.getValue()) + .build(); + builder.addMetrics(nameInt64Pair); + } + return builder.build(); + } + /** * Unwraps an exception from a protobuf service into the underlying (expected) IOException. * This method will always throw an exception. diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java index 8e48cb5..6cf1594 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java @@ -4685,7 +4685,7 @@ public class TestFromClientSide { ScanMetrics scanMetrics = getScanMetrics(scan); assertEquals("Did not access all the regions in the table", numOfRegions, - scanMetrics.countOfRegions.getCurrentIntervalValue()); + scanMetrics.countOfRegions.get()); // now, test that the metrics are still collected even if you don't call close, but do // run past the end of all the records @@ -4697,7 +4697,7 @@ public class TestFromClientSide { } ScanMetrics scanMetricsWithoutClose = getScanMetrics(scanWithoutClose); assertEquals("Did not access all the regions in the table", numOfRegions, - scanMetricsWithoutClose.countOfRegions.getCurrentIntervalValue()); + scanMetricsWithoutClose.countOfRegions.get()); // finally, test that the metrics are collected correctly if you both run past all the records, // AND close the scanner @@ -4711,17 +4711,16 @@ public class TestFromClientSide { scannerWithClose.close(); ScanMetrics scanMetricsWithClose = getScanMetrics(scanWithClose); assertEquals("Did not access all the regions in the table", numOfRegions, - scanMetricsWithClose.countOfRegions.getCurrentIntervalValue()); + scanMetricsWithClose.countOfRegions.get()); } private ScanMetrics getScanMetrics(Scan scan) throws Exception { byte[] serializedMetrics = scan.getAttribute(Scan.SCAN_ATTRIBUTES_METRICS_DATA); assertTrue("Serialized metrics were not found.", serializedMetrics != null); - DataInputBuffer in = new DataInputBuffer(); - in.reset(serializedMetrics, 0, serializedMetrics.length); - ScanMetrics scanMetrics = new ScanMetrics(); - scanMetrics.readFields(in); + + ScanMetrics scanMetrics = ProtobufUtil.toScanMetrics(serializedMetrics); + return scanMetrics; } -- 1.7.10.2 (Apple Git-33)