diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTable.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTable.java index 6ba0b87..5ed8c79 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTable.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTable.java @@ -595,7 +595,7 @@ public class HTable implements HTableInterface { try { ClientProtos.GetResponse response = getStub().get(controller, request); if (!response.hasResult()) return null; - return ProtobufUtil.toResult(response.getResult()); + return ProtobufUtil.toResult(response.getResult(), controller.cellScanner()); } catch (ServiceException se) { throw ProtobufUtil.getRemoteException(se); } @@ -700,7 +700,7 @@ public class HTable implements HTableInterface { try { ClientProtos.GetResponse response = getStub().get(controller, request); if (response == null) return null; - return ProtobufUtil.toResult(response.getResult()); + return ProtobufUtil.toResult(response.getResult(), controller.cellScanner()); } catch (ServiceException se) { throw ProtobufUtil.getRemoteException(se); } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCallerWithReadReplicas.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCallerWithReadReplicas.java index 8f28881..f587a96 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCallerWithReadReplicas.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCallerWithReadReplicas.java @@ -161,7 +161,7 @@ public class RpcRetryingCallerWithReadReplicas { if (response == null) { return null; } - return ProtobufUtil.toResult(response.getResult()); + return ProtobufUtil.toResult(response.getResult(), controller.cellScanner()); } catch (ServiceException se) { throw ProtobufUtil.getRemoteException(se); } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java index cda6bd7..ac3e3d5 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java @@ -37,6 +37,7 @@ import java.util.Map.Entry; import java.util.NavigableSet; import java.util.concurrent.TimeUnit; +import org.apache.commons.net.ftp.parser.ParserInitializationException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; @@ -3051,7 +3052,13 @@ public final class ProtobufUtil { */ public static RPCProtos.VersionInfo getVersionInfo() { RPCProtos.VersionInfo.Builder builder = RPCProtos.VersionInfo.newBuilder(); - builder.setVersion(VersionInfo.getVersion()); + String version = VersionInfo.getVersion(); + builder.setVersion(version); + String[] components = version.split("\\."); + if (components != null && components.length > 2) { + builder.setVersionMajor(Integer.parseInt(components[0])); + builder.setVersionMinor(Integer.parseInt(components[1])); + } builder.setUrl(VersionInfo.getUrl()); builder.setRevision(VersionInfo.getRevision()); builder.setUser(VersionInfo.getUser()); @@ -3059,4 +3066,4 @@ public final class ProtobufUtil { builder.setSrcChecksum(VersionInfo.getSrcChecksum()); return builder.build(); } -} +} \ No newline at end of file diff --git a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/RPCProtos.java b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/RPCProtos.java index 7758e98..195e0f1 100644 --- a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/RPCProtos.java +++ b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/RPCProtos.java @@ -790,6 +790,26 @@ public final class RPCProtos { */ com.google.protobuf.ByteString getSrcChecksumBytes(); + + // optional uint32 version_major = 7; + /** + * optional uint32 version_major = 7; + */ + boolean hasVersionMajor(); + /** + * optional uint32 version_major = 7; + */ + int getVersionMajor(); + + // optional uint32 version_minor = 8; + /** + * optional uint32 version_minor = 8; + */ + boolean hasVersionMinor(); + /** + * optional uint32 version_minor = 8; + */ + int getVersionMinor(); } /** * Protobuf type {@code VersionInfo} @@ -876,6 +896,16 @@ public final class RPCProtos { srcChecksum_ = input.readBytes(); break; } + case 56: { + bitField0_ |= 0x00000040; + versionMajor_ = input.readUInt32(); + break; + } + case 64: { + bitField0_ |= 0x00000080; + versionMinor_ = input.readUInt32(); + break; + } } } } catch (com.google.protobuf.InvalidProtocolBufferException e) { @@ -1174,6 +1204,38 @@ public final class RPCProtos { } } + // optional uint32 version_major = 7; + public static final int VERSION_MAJOR_FIELD_NUMBER = 7; + private int versionMajor_; + /** + * optional uint32 version_major = 7; + */ + public boolean hasVersionMajor() { + return ((bitField0_ & 0x00000040) == 0x00000040); + } + /** + * optional uint32 version_major = 7; + */ + public int getVersionMajor() { + return versionMajor_; + } + + // optional uint32 version_minor = 8; + public static final int VERSION_MINOR_FIELD_NUMBER = 8; + private int versionMinor_; + /** + * optional uint32 version_minor = 8; + */ + public boolean hasVersionMinor() { + return ((bitField0_ & 0x00000080) == 0x00000080); + } + /** + * optional uint32 version_minor = 8; + */ + public int getVersionMinor() { + return versionMinor_; + } + private void initFields() { version_ = ""; url_ = ""; @@ -1181,6 +1243,8 @@ public final class RPCProtos { user_ = ""; date_ = ""; srcChecksum_ = ""; + versionMajor_ = 0; + versionMinor_ = 0; } private byte memoizedIsInitialized = -1; public final boolean isInitialized() { @@ -1236,6 +1300,12 @@ public final class RPCProtos { if (((bitField0_ & 0x00000020) == 0x00000020)) { output.writeBytes(6, getSrcChecksumBytes()); } + if (((bitField0_ & 0x00000040) == 0x00000040)) { + output.writeUInt32(7, versionMajor_); + } + if (((bitField0_ & 0x00000080) == 0x00000080)) { + output.writeUInt32(8, versionMinor_); + } getUnknownFields().writeTo(output); } @@ -1269,6 +1339,14 @@ public final class RPCProtos { size += com.google.protobuf.CodedOutputStream .computeBytesSize(6, getSrcChecksumBytes()); } + if (((bitField0_ & 0x00000040) == 0x00000040)) { + size += com.google.protobuf.CodedOutputStream + .computeUInt32Size(7, versionMajor_); + } + if (((bitField0_ & 0x00000080) == 0x00000080)) { + size += com.google.protobuf.CodedOutputStream + .computeUInt32Size(8, versionMinor_); + } size += getUnknownFields().getSerializedSize(); memoizedSerializedSize = size; return size; @@ -1322,6 +1400,16 @@ public final class RPCProtos { result = result && getSrcChecksum() .equals(other.getSrcChecksum()); } + result = result && (hasVersionMajor() == other.hasVersionMajor()); + if (hasVersionMajor()) { + result = result && (getVersionMajor() + == other.getVersionMajor()); + } + result = result && (hasVersionMinor() == other.hasVersionMinor()); + if (hasVersionMinor()) { + result = result && (getVersionMinor() + == other.getVersionMinor()); + } result = result && getUnknownFields().equals(other.getUnknownFields()); return result; @@ -1359,6 +1447,14 @@ public final class RPCProtos { hash = (37 * hash) + SRC_CHECKSUM_FIELD_NUMBER; hash = (53 * hash) + getSrcChecksum().hashCode(); } + if (hasVersionMajor()) { + hash = (37 * hash) + VERSION_MAJOR_FIELD_NUMBER; + hash = (53 * hash) + getVersionMajor(); + } + if (hasVersionMinor()) { + hash = (37 * hash) + VERSION_MINOR_FIELD_NUMBER; + hash = (53 * hash) + getVersionMinor(); + } hash = (29 * hash) + getUnknownFields().hashCode(); memoizedHashCode = hash; return hash; @@ -1484,6 +1580,10 @@ public final class RPCProtos { bitField0_ = (bitField0_ & ~0x00000010); srcChecksum_ = ""; bitField0_ = (bitField0_ & ~0x00000020); + versionMajor_ = 0; + bitField0_ = (bitField0_ & ~0x00000040); + versionMinor_ = 0; + bitField0_ = (bitField0_ & ~0x00000080); return this; } @@ -1536,6 +1636,14 @@ public final class RPCProtos { to_bitField0_ |= 0x00000020; } result.srcChecksum_ = srcChecksum_; + if (((from_bitField0_ & 0x00000040) == 0x00000040)) { + to_bitField0_ |= 0x00000040; + } + result.versionMajor_ = versionMajor_; + if (((from_bitField0_ & 0x00000080) == 0x00000080)) { + to_bitField0_ |= 0x00000080; + } + result.versionMinor_ = versionMinor_; result.bitField0_ = to_bitField0_; onBuilt(); return result; @@ -1582,6 +1690,12 @@ public final class RPCProtos { srcChecksum_ = other.srcChecksum_; onChanged(); } + if (other.hasVersionMajor()) { + setVersionMajor(other.getVersionMajor()); + } + if (other.hasVersionMinor()) { + setVersionMinor(other.getVersionMinor()); + } this.mergeUnknownFields(other.getUnknownFields()); return this; } @@ -2077,6 +2191,72 @@ public final class RPCProtos { return this; } + // optional uint32 version_major = 7; + private int versionMajor_ ; + /** + * optional uint32 version_major = 7; + */ + public boolean hasVersionMajor() { + return ((bitField0_ & 0x00000040) == 0x00000040); + } + /** + * optional uint32 version_major = 7; + */ + public int getVersionMajor() { + return versionMajor_; + } + /** + * optional uint32 version_major = 7; + */ + public Builder setVersionMajor(int value) { + bitField0_ |= 0x00000040; + versionMajor_ = value; + onChanged(); + return this; + } + /** + * optional uint32 version_major = 7; + */ + public Builder clearVersionMajor() { + bitField0_ = (bitField0_ & ~0x00000040); + versionMajor_ = 0; + onChanged(); + return this; + } + + // optional uint32 version_minor = 8; + private int versionMinor_ ; + /** + * optional uint32 version_minor = 8; + */ + public boolean hasVersionMinor() { + return ((bitField0_ & 0x00000080) == 0x00000080); + } + /** + * optional uint32 version_minor = 8; + */ + public int getVersionMinor() { + return versionMinor_; + } + /** + * optional uint32 version_minor = 8; + */ + public Builder setVersionMinor(int value) { + bitField0_ |= 0x00000080; + versionMinor_ = value; + onChanged(); + return this; + } + /** + * optional uint32 version_minor = 8; + */ + public Builder clearVersionMinor() { + bitField0_ = (bitField0_ & ~0x00000080); + versionMinor_ = 0; + onChanged(); + return this; + } + // @@protoc_insertion_point(builder_scope:VersionInfo) } @@ -7527,27 +7707,28 @@ public final class RPCProtos { java.lang.String[] descriptorData = { "\n\tRPC.proto\032\rTracing.proto\032\013HBase.proto\"" + "<\n\017UserInformation\022\026\n\016effective_user\030\001 \002" + - "(\t\022\021\n\treal_user\030\002 \001(\t\"o\n\013VersionInfo\022\017\n\007" + - "version\030\001 \002(\t\022\013\n\003url\030\002 \002(\t\022\020\n\010revision\030\003" + - " \002(\t\022\014\n\004user\030\004 \002(\t\022\014\n\004date\030\005 \002(\t\022\024\n\014src_" + - "checksum\030\006 \002(\t\"\266\001\n\020ConnectionHeader\022#\n\tu" + - "ser_info\030\001 \001(\0132\020.UserInformation\022\024\n\014serv" + - "ice_name\030\002 \001(\t\022\036\n\026cell_block_codec_class" + - "\030\003 \001(\t\022#\n\033cell_block_compressor_class\030\004 " + - "\001(\t\022\"\n\014version_info\030\005 \001(\0132\014.VersionInfo\"", - "\037\n\rCellBlockMeta\022\016\n\006length\030\001 \001(\r\"|\n\021Exce" + - "ptionResponse\022\034\n\024exception_class_name\030\001 " + - "\001(\t\022\023\n\013stack_trace\030\002 \001(\t\022\020\n\010hostname\030\003 \001" + - "(\t\022\014\n\004port\030\004 \001(\005\022\024\n\014do_not_retry\030\005 \001(\010\"\246" + - "\001\n\rRequestHeader\022\017\n\007call_id\030\001 \001(\r\022\035\n\ntra" + - "ce_info\030\002 \001(\0132\t.RPCTInfo\022\023\n\013method_name\030" + - "\003 \001(\t\022\025\n\rrequest_param\030\004 \001(\010\022\'\n\017cell_blo" + - "ck_meta\030\005 \001(\0132\016.CellBlockMeta\022\020\n\010priorit" + - "y\030\006 \001(\r\"q\n\016ResponseHeader\022\017\n\007call_id\030\001 \001" + - "(\r\022%\n\texception\030\002 \001(\0132\022.ExceptionRespons", - "e\022\'\n\017cell_block_meta\030\003 \001(\0132\016.CellBlockMe" + - "taB<\n*org.apache.hadoop.hbase.protobuf.g" + - "eneratedB\tRPCProtosH\001\240\001\001" + "(\t\022\021\n\treal_user\030\002 \001(\t\"\235\001\n\013VersionInfo\022\017\n" + + "\007version\030\001 \002(\t\022\013\n\003url\030\002 \002(\t\022\020\n\010revision\030" + + "\003 \002(\t\022\014\n\004user\030\004 \002(\t\022\014\n\004date\030\005 \002(\t\022\024\n\014src" + + "_checksum\030\006 \002(\t\022\025\n\rversion_major\030\007 \001(\r\022\025" + + "\n\rversion_minor\030\010 \001(\r\"\266\001\n\020ConnectionHead" + + "er\022#\n\tuser_info\030\001 \001(\0132\020.UserInformation\022" + + "\024\n\014service_name\030\002 \001(\t\022\036\n\026cell_block_code" + + "c_class\030\003 \001(\t\022#\n\033cell_block_compressor_c", + "lass\030\004 \001(\t\022\"\n\014version_info\030\005 \001(\0132\014.Versi" + + "onInfo\"\037\n\rCellBlockMeta\022\016\n\006length\030\001 \001(\r\"" + + "|\n\021ExceptionResponse\022\034\n\024exception_class_" + + "name\030\001 \001(\t\022\023\n\013stack_trace\030\002 \001(\t\022\020\n\010hostn" + + "ame\030\003 \001(\t\022\014\n\004port\030\004 \001(\005\022\024\n\014do_not_retry\030" + + "\005 \001(\010\"\246\001\n\rRequestHeader\022\017\n\007call_id\030\001 \001(\r" + + "\022\035\n\ntrace_info\030\002 \001(\0132\t.RPCTInfo\022\023\n\013metho" + + "d_name\030\003 \001(\t\022\025\n\rrequest_param\030\004 \001(\010\022\'\n\017c" + + "ell_block_meta\030\005 \001(\0132\016.CellBlockMeta\022\020\n\010" + + "priority\030\006 \001(\r\"q\n\016ResponseHeader\022\017\n\007call", + "_id\030\001 \001(\r\022%\n\texception\030\002 \001(\0132\022.Exception" + + "Response\022\'\n\017cell_block_meta\030\003 \001(\0132\016.Cell" + + "BlockMetaB<\n*org.apache.hadoop.hbase.pro" + + "tobuf.generatedB\tRPCProtosH\001\240\001\001" }; com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { @@ -7565,7 +7746,7 @@ public final class RPCProtos { internal_static_VersionInfo_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_VersionInfo_descriptor, - new java.lang.String[] { "Version", "Url", "Revision", "User", "Date", "SrcChecksum", }); + new java.lang.String[] { "Version", "Url", "Revision", "User", "Date", "SrcChecksum", "VersionMajor", "VersionMinor", }); internal_static_ConnectionHeader_descriptor = getDescriptor().getMessageTypes().get(2); internal_static_ConnectionHeader_fieldAccessorTable = new diff --git a/hbase-protocol/src/main/protobuf/RPC.proto b/hbase-protocol/src/main/protobuf/RPC.proto index a5d60d8..84950e0 100644 --- a/hbase-protocol/src/main/protobuf/RPC.proto +++ b/hbase-protocol/src/main/protobuf/RPC.proto @@ -84,6 +84,8 @@ message VersionInfo { required string user = 4; required string date = 5; required string src_checksum = 6; + optional uint32 version_major = 7; + optional uint32 version_minor = 8; } // This is sent on connection setup after the connection preamble is sent. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ProcedurePrepareLatch.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ProcedurePrepareLatch.java index 2a1abca..6da4dc0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ProcedurePrepareLatch.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ProcedurePrepareLatch.java @@ -24,10 +24,8 @@ import java.util.concurrent.CountDownLatch; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; -import org.apache.hadoop.hbase.ipc.RpcServer; -import org.apache.hadoop.hbase.ipc.RpcCallContext; import org.apache.hadoop.hbase.procedure2.Procedure; -import org.apache.hadoop.hbase.protobuf.generated.RPCProtos.VersionInfo; +import org.apache.hadoop.hbase.util.VersionUtil; /** * Latch used by the Master to have the prepare() sync behaviour for old @@ -44,24 +42,7 @@ public abstract class ProcedurePrepareLatch { } public static boolean hasProcedureSupport() { - return currentClientHasMinimumVersion(1, 1); - } - - private static boolean currentClientHasMinimumVersion(int major, int minor) { - RpcCallContext call = RpcServer.getCurrentCall(); - VersionInfo versionInfo = call != null ? call.getClientVersionInfo() : null; - if (versionInfo != null) { - String[] components = versionInfo.getVersion().split("\\."); - - int clientMajor = components.length > 0 ? Integer.parseInt(components[0]) : 0; - if (clientMajor != major) { - return clientMajor > major; - } - - int clientMinor = components.length > 1 ? Integer.parseInt(components[1]) : 0; - return clientMinor >= minor; - } - return false; + return VersionUtil.currentClientHasMinimumVersion(1, 1); } protected abstract void countDown(final Procedure proc); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java index 6491a5c..68c389b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java @@ -164,6 +164,7 @@ import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.util.ServerRegionReplicaUtil; import org.apache.hadoop.hbase.util.Strings; +import org.apache.hadoop.hbase.util.VersionUtil; import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.wal.WALKey; import org.apache.hadoop.hbase.wal.WALSplitter; @@ -1888,7 +1889,15 @@ public class RSRpcServices implements HBaseRPCErrorHandler, ProtobufUtil.toResult(existence, region.getRegionInfo().getReplicaId() != 0); builder.setResult(pbr); } else if (r != null) { - ClientProtos.Result pbr = ProtobufUtil.toResult(r); + ClientProtos.Result pbr; + if (isClientCellBlockSupport() && controller instanceof PayloadCarryingRpcController + && VersionUtil.currentClientHasMinimumVersion(1, 2)) { + pbr = ProtobufUtil.toResultNoData(r); + ((PayloadCarryingRpcController) controller).setCellScanner(CellUtil.createCellScanner(r + .rawCells())); + } else { + pbr = ProtobufUtil.toResult(r); + } builder.setResult(pbr); } if (r != null) {