diff --git a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/WALProtos.java b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/WALProtos.java index 3d0d1d0..b1ab3c3 100644 --- a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/WALProtos.java +++ b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/WALProtos.java @@ -137,6 +137,21 @@ public final class WALProtos { */ com.google.protobuf.ByteString getWriterClsNameBytes(); + + // optional string codec_cls_name = 5; + /** + * optional string codec_cls_name = 5; + */ + boolean hasCodecClsName(); + /** + * optional string codec_cls_name = 5; + */ + java.lang.String getCodecClsName(); + /** + * optional string codec_cls_name = 5; + */ + com.google.protobuf.ByteString + getCodecClsNameBytes(); } /** * Protobuf type {@code WALHeader} @@ -209,6 +224,11 @@ public final class WALProtos { writerClsName_ = input.readBytes(); break; } + case 42: { + bitField0_ |= 0x00000010; + codecClsName_ = input.readBytes(); + break; + } } } } catch (com.google.protobuf.InvalidProtocolBufferException e) { @@ -340,11 +360,55 @@ public final class WALProtos { } } + // optional string codec_cls_name = 5; + public static final int CODEC_CLS_NAME_FIELD_NUMBER = 5; + private java.lang.Object codecClsName_; + /** + * optional string codec_cls_name = 5; + */ + public boolean hasCodecClsName() { + return ((bitField0_ & 0x00000010) == 0x00000010); + } + /** + * optional string codec_cls_name = 5; + */ + public java.lang.String getCodecClsName() { + java.lang.Object ref = codecClsName_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + codecClsName_ = s; + } + return s; + } + } + /** + * optional string codec_cls_name = 5; + */ + public com.google.protobuf.ByteString + getCodecClsNameBytes() { + java.lang.Object ref = codecClsName_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8( + (java.lang.String) ref); + codecClsName_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + private void initFields() { hasCompression_ = false; encryptionKey_ = com.google.protobuf.ByteString.EMPTY; hasTagCompression_ = false; writerClsName_ = ""; + codecClsName_ = ""; } private byte memoizedIsInitialized = -1; public final boolean isInitialized() { @@ -370,6 +434,9 @@ public final class WALProtos { if (((bitField0_ & 0x00000008) == 0x00000008)) { output.writeBytes(4, getWriterClsNameBytes()); } + if (((bitField0_ & 0x00000010) == 0x00000010)) { + output.writeBytes(5, getCodecClsNameBytes()); + } getUnknownFields().writeTo(output); } @@ -395,6 +462,10 @@ public final class WALProtos { size += com.google.protobuf.CodedOutputStream .computeBytesSize(4, getWriterClsNameBytes()); } + if (((bitField0_ & 0x00000010) == 0x00000010)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(5, getCodecClsNameBytes()); + } size += getUnknownFields().getSerializedSize(); memoizedSerializedSize = size; return size; @@ -438,6 +509,11 @@ public final class WALProtos { result = result && getWriterClsName() .equals(other.getWriterClsName()); } + result = result && (hasCodecClsName() == other.hasCodecClsName()); + if (hasCodecClsName()) { + result = result && getCodecClsName() + .equals(other.getCodecClsName()); + } result = result && getUnknownFields().equals(other.getUnknownFields()); return result; @@ -467,6 +543,10 @@ public final class WALProtos { hash = (37 * hash) + WRITER_CLS_NAME_FIELD_NUMBER; hash = (53 * hash) + getWriterClsName().hashCode(); } + if (hasCodecClsName()) { + hash = (37 * hash) + CODEC_CLS_NAME_FIELD_NUMBER; + hash = (53 * hash) + getCodecClsName().hashCode(); + } hash = (29 * hash) + getUnknownFields().hashCode(); memoizedHashCode = hash; return hash; @@ -584,6 +664,8 @@ public final class WALProtos { bitField0_ = (bitField0_ & ~0x00000004); writerClsName_ = ""; bitField0_ = (bitField0_ & ~0x00000008); + codecClsName_ = ""; + bitField0_ = (bitField0_ & ~0x00000010); return this; } @@ -628,6 +710,10 @@ public final class WALProtos { to_bitField0_ |= 0x00000008; } result.writerClsName_ = writerClsName_; + if (((from_bitField0_ & 0x00000010) == 0x00000010)) { + to_bitField0_ |= 0x00000010; + } + result.codecClsName_ = codecClsName_; result.bitField0_ = to_bitField0_; onBuilt(); return result; @@ -658,6 +744,11 @@ public final class WALProtos { writerClsName_ = other.writerClsName_; onChanged(); } + if (other.hasCodecClsName()) { + bitField0_ |= 0x00000010; + codecClsName_ = other.codecClsName_; + onChanged(); + } this.mergeUnknownFields(other.getUnknownFields()); return this; } @@ -861,6 +952,80 @@ public final class WALProtos { return this; } + // optional string codec_cls_name = 5; + private java.lang.Object codecClsName_ = ""; + /** + * optional string codec_cls_name = 5; + */ + public boolean hasCodecClsName() { + return ((bitField0_ & 0x00000010) == 0x00000010); + } + /** + * optional string codec_cls_name = 5; + */ + public java.lang.String getCodecClsName() { + java.lang.Object ref = codecClsName_; + if (!(ref instanceof java.lang.String)) { + java.lang.String s = ((com.google.protobuf.ByteString) ref) + .toStringUtf8(); + codecClsName_ = s; + return s; + } else { + return (java.lang.String) ref; + } + } + /** + * optional string codec_cls_name = 5; + */ + public com.google.protobuf.ByteString + getCodecClsNameBytes() { + java.lang.Object ref = codecClsName_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8( + (java.lang.String) ref); + codecClsName_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + /** + * optional string codec_cls_name = 5; + */ + public Builder setCodecClsName( + java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000010; + codecClsName_ = value; + onChanged(); + return this; + } + /** + * optional string codec_cls_name = 5; + */ + public Builder clearCodecClsName() { + bitField0_ = (bitField0_ & ~0x00000010); + codecClsName_ = getDefaultInstance().getCodecClsName(); + onChanged(); + return this; + } + /** + * optional string codec_cls_name = 5; + */ + public Builder setCodecClsNameBytes( + com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000010; + codecClsName_ = value; + onChanged(); + return this; + } + // @@protoc_insertion_point(builder_scope:WALHeader) } @@ -7829,37 +7994,37 @@ public final class WALProtos { descriptor; static { java.lang.String[] descriptorData = { - "\n\tWAL.proto\032\013HBase.proto\"r\n\tWALHeader\022\027\n" + - "\017has_compression\030\001 \001(\010\022\026\n\016encryption_key" + - "\030\002 \001(\014\022\033\n\023has_tag_compression\030\003 \001(\010\022\027\n\017w" + - "riter_cls_name\030\004 \001(\t\"\240\002\n\006WALKey\022\033\n\023encod" + - "ed_region_name\030\001 \002(\014\022\022\n\ntable_name\030\002 \002(\014" + - "\022\033\n\023log_sequence_number\030\003 \002(\004\022\022\n\nwrite_t" + - "ime\030\004 \002(\004\022\035\n\ncluster_id\030\005 \001(\0132\005.UUIDB\002\030\001" + - "\022\034\n\006scopes\030\006 \003(\0132\014.FamilyScope\022\032\n\022follow" + - "ing_kv_count\030\007 \001(\r\022\032\n\013cluster_ids\030\010 \003(\0132" + - "\005.UUID\022\022\n\nnonceGroup\030\t \001(\004\022\r\n\005nonce\030\n \001(", - "\004\022\034\n\024orig_sequence_number\030\013 \001(\004\"=\n\013Famil" + - "yScope\022\016\n\006family\030\001 \002(\014\022\036\n\nscope_type\030\002 \002" + - "(\0162\n.ScopeType\"\276\001\n\024CompactionDescriptor\022" + - "\022\n\ntable_name\030\001 \002(\014\022\033\n\023encoded_region_na" + - "me\030\002 \002(\014\022\023\n\013family_name\030\003 \002(\014\022\030\n\020compact" + - "ion_input\030\004 \003(\t\022\031\n\021compaction_output\030\005 \003" + - "(\t\022\026\n\016store_home_dir\030\006 \002(\t\022\023\n\013region_nam" + - "e\030\007 \001(\014\"\353\002\n\017FlushDescriptor\022,\n\006action\030\001 " + - "\002(\0162\034.FlushDescriptor.FlushAction\022\022\n\ntab" + - "le_name\030\002 \002(\014\022\033\n\023encoded_region_name\030\003 \002", - "(\014\022\035\n\025flush_sequence_number\030\004 \001(\004\022<\n\rsto" + - "re_flushes\030\005 \003(\0132%.FlushDescriptor.Store" + - "FlushDescriptor\032Y\n\024StoreFlushDescriptor\022" + - "\023\n\013family_name\030\001 \002(\014\022\026\n\016store_home_dir\030\002" + - " \002(\t\022\024\n\014flush_output\030\003 \003(\t\"A\n\013FlushActio" + - "n\022\017\n\013START_FLUSH\020\000\022\020\n\014COMMIT_FLUSH\020\001\022\017\n\013" + - "ABORT_FLUSH\020\002\"\014\n\nWALTrailer*F\n\tScopeType" + - "\022\033\n\027REPLICATION_SCOPE_LOCAL\020\000\022\034\n\030REPLICA" + - "TION_SCOPE_GLOBAL\020\001B?\n*org.apache.hadoop" + - ".hbase.protobuf.generatedB\tWALProtosH\001\210\001", - "\000\240\001\001" + "\n\tWAL.proto\032\013HBase.proto\"\212\001\n\tWALHeader\022\027" + + "\n\017has_compression\030\001 \001(\010\022\026\n\016encryption_ke" + + "y\030\002 \001(\014\022\033\n\023has_tag_compression\030\003 \001(\010\022\027\n\017" + + "writer_cls_name\030\004 \001(\t\022\026\n\016codec_cls_name\030" + + "\005 \001(\t\"\240\002\n\006WALKey\022\033\n\023encoded_region_name\030" + + "\001 \002(\014\022\022\n\ntable_name\030\002 \002(\014\022\033\n\023log_sequenc" + + "e_number\030\003 \002(\004\022\022\n\nwrite_time\030\004 \002(\004\022\035\n\ncl" + + "uster_id\030\005 \001(\0132\005.UUIDB\002\030\001\022\034\n\006scopes\030\006 \003(" + + "\0132\014.FamilyScope\022\032\n\022following_kv_count\030\007 " + + "\001(\r\022\032\n\013cluster_ids\030\010 \003(\0132\005.UUID\022\022\n\nnonce", + "Group\030\t \001(\004\022\r\n\005nonce\030\n \001(\004\022\034\n\024orig_seque" + + "nce_number\030\013 \001(\004\"=\n\013FamilyScope\022\016\n\006famil" + + "y\030\001 \002(\014\022\036\n\nscope_type\030\002 \002(\0162\n.ScopeType\"" + + "\276\001\n\024CompactionDescriptor\022\022\n\ntable_name\030\001" + + " \002(\014\022\033\n\023encoded_region_name\030\002 \002(\014\022\023\n\013fam" + + "ily_name\030\003 \002(\014\022\030\n\020compaction_input\030\004 \003(\t" + + "\022\031\n\021compaction_output\030\005 \003(\t\022\026\n\016store_hom" + + "e_dir\030\006 \002(\t\022\023\n\013region_name\030\007 \001(\014\"\353\002\n\017Flu" + + "shDescriptor\022,\n\006action\030\001 \002(\0162\034.FlushDesc" + + "riptor.FlushAction\022\022\n\ntable_name\030\002 \002(\014\022\033", + "\n\023encoded_region_name\030\003 \002(\014\022\035\n\025flush_seq" + + "uence_number\030\004 \001(\004\022<\n\rstore_flushes\030\005 \003(" + + "\0132%.FlushDescriptor.StoreFlushDescriptor" + + "\032Y\n\024StoreFlushDescriptor\022\023\n\013family_name\030" + + "\001 \002(\014\022\026\n\016store_home_dir\030\002 \002(\t\022\024\n\014flush_o" + + "utput\030\003 \003(\t\"A\n\013FlushAction\022\017\n\013START_FLUS" + + "H\020\000\022\020\n\014COMMIT_FLUSH\020\001\022\017\n\013ABORT_FLUSH\020\002\"\014" + + "\n\nWALTrailer*F\n\tScopeType\022\033\n\027REPLICATION" + + "_SCOPE_LOCAL\020\000\022\034\n\030REPLICATION_SCOPE_GLOB" + + "AL\020\001B?\n*org.apache.hadoop.hbase.protobuf", + ".generatedB\tWALProtosH\001\210\001\000\240\001\001" }; com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { @@ -7871,7 +8036,7 @@ public final class WALProtos { internal_static_WALHeader_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_WALHeader_descriptor, - new java.lang.String[] { "HasCompression", "EncryptionKey", "HasTagCompression", "WriterClsName", }); + new java.lang.String[] { "HasCompression", "EncryptionKey", "HasTagCompression", "WriterClsName", "CodecClsName", }); internal_static_WALKey_descriptor = getDescriptor().getMessageTypes().get(1); internal_static_WALKey_fieldAccessorTable = new diff --git a/hbase-protocol/src/main/protobuf/WAL.proto b/hbase-protocol/src/main/protobuf/WAL.proto index 6c05c75..80207a6 100644 --- a/hbase-protocol/src/main/protobuf/WAL.proto +++ b/hbase-protocol/src/main/protobuf/WAL.proto @@ -28,6 +28,7 @@ message WALHeader { optional bytes encryption_key = 2; optional bool has_tag_compression = 3; optional string writer_cls_name = 4; + optional string codec_cls_name = 5; } // Protocol buffer version of HLogKey; see HLogKey comment, not really a key but WALEdit header for some KVs diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogWriter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogWriter.java index 74552f4..ff6135f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogWriter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogWriter.java @@ -58,10 +58,14 @@ public class ProtobufLogWriter extends WriterBase { return WALCellCodec.create(conf, compressionContext); } - protected WALHeader buildWALHeader(WALHeader.Builder builder) throws IOException { + protected WALHeader buildWALHeader(Configuration conf, WALHeader.Builder builder) + throws IOException { if (!builder.hasWriterClsName()) { builder.setWriterClsName(ProtobufLogWriter.class.getSimpleName()); } + if (!builder.hasCodecClsName()) { + builder.setCodecClsName(WALCellCodec.getWALCellCodecClass(conf)); + } return builder.build(); } @@ -83,7 +87,7 @@ public class ProtobufLogWriter extends WriterBase { output.write(ProtobufLogReader.PB_WAL_MAGIC); boolean doTagCompress = doCompress && conf.getBoolean(CompressionContext.ENABLE_WAL_TAGS_COMPRESSION, true); - buildWALHeader( + buildWALHeader(conf, WALHeader.newBuilder().setHasCompression(doCompress).setHasTagCompression(doTagCompress)) .writeDelimitedTo(output); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALCellCodec.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALCellCodec.java index 3653884..c16638c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALCellCodec.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALCellCodec.java @@ -77,6 +77,10 @@ public class WALCellCodec implements Codec { this.compression = compression; } + static String getWALCellCodecClass(Configuration conf) { + return conf.get(WAL_CELL_CODEC_CLASS_KEY, WALCellCodec.class.getName()); + } + /** * Create and setup a {@link WALCellCodec} from the {@link Configuration} and CompressionContext, * if they have been specified. Fully prepares the codec for use. @@ -88,8 +92,7 @@ public class WALCellCodec implements Codec { */ public static WALCellCodec create(Configuration conf, CompressionContext compression) throws UnsupportedOperationException { - String className = conf.get(WAL_CELL_CODEC_CLASS_KEY, WALCellCodec.class.getName()); - return ReflectionUtils.instantiateWithCustomCtor(className, new Class[] { Configuration.class, + return ReflectionUtils.instantiateWithCustomCtor(getWALCellCodecClass(conf), new Class[] { Configuration.class, CompressionContext.class }, new Object[] { conf, compression }); }