From 29ed76fa04ff82e10a202fb02c9d7964ece2603c Mon Sep 17 00:00:00 2001 From: Elliott Clark Date: Mon, 28 Dec 2015 15:33:52 -0800 Subject: [PATCH] HBASE-14853 Add on protobuf to c++ chain --- hbase-native-client/core/BUCK | 1 + hbase-native-client/core/client.cc | 4 + hbase-native-client/core/client.h | 2 + hbase-native-client/if/AccessControl.proto | 123 ++++ hbase-native-client/if/Admin.proto | 309 ++++++++++ hbase-native-client/if/Aggregate.proto | 63 ++ hbase-native-client/if/Authentication.proto | 82 +++ hbase-native-client/if/BUCK | 36 ++ hbase-native-client/if/Cell.proto | 67 ++ hbase-native-client/if/Client.proto | 472 ++++++++++++++ hbase-native-client/if/ClusterId.proto | 34 ++ hbase-native-client/if/ClusterStatus.proto | 224 +++++++ hbase-native-client/if/Comparator.proto | 74 +++ hbase-native-client/if/Encryption.proto | 33 + hbase-native-client/if/ErrorHandling.proto | 58 ++ hbase-native-client/if/FS.proto | 45 ++ hbase-native-client/if/Filter.proto | 170 ++++++ hbase-native-client/if/HBase.proto | 258 ++++++++ hbase-native-client/if/HFile.proto | 49 ++ hbase-native-client/if/LoadBalancer.proto | 29 + hbase-native-client/if/MapReduce.proto | 37 ++ hbase-native-client/if/Master.proto | 778 ++++++++++++++++++++++++ hbase-native-client/if/MasterProcedure.proto | 245 ++++++++ hbase-native-client/if/MultiRowMutation.proto | 45 ++ hbase-native-client/if/Procedure.proto | 119 ++++ hbase-native-client/if/Quota.proto | 76 +++ hbase-native-client/if/RPC.proto | 136 +++++ hbase-native-client/if/RegionNormalizer.proto | 28 + hbase-native-client/if/RegionServerStatus.proto | 158 +++++ hbase-native-client/if/RowProcessor.proto | 45 ++ hbase-native-client/if/SecureBulkLoad.proto | 72 +++ hbase-native-client/if/Snapshot.proto | 66 ++ hbase-native-client/if/Tracing.proto | 33 + hbase-native-client/if/VisibilityLabels.proto | 83 +++ hbase-native-client/if/WAL.proto | 172 ++++++ hbase-native-client/if/ZooKeeper.proto | 155 +++++ hbase-native-client/rpc/CMakeLists.txt | 17 - hbase-native-client/third-party/BUCK | 4 +- 38 files changed, 4384 insertions(+), 18 deletions(-) create mode 100644 hbase-native-client/if/AccessControl.proto create mode 100644 hbase-native-client/if/Admin.proto create mode 100644 hbase-native-client/if/Aggregate.proto create mode 100644 hbase-native-client/if/Authentication.proto create mode 100644 hbase-native-client/if/BUCK create mode 100644 hbase-native-client/if/Cell.proto create mode 100644 hbase-native-client/if/Client.proto create mode 100644 hbase-native-client/if/ClusterId.proto create mode 100644 hbase-native-client/if/ClusterStatus.proto create mode 100644 hbase-native-client/if/Comparator.proto create mode 100644 hbase-native-client/if/Encryption.proto create mode 100644 hbase-native-client/if/ErrorHandling.proto create mode 100644 hbase-native-client/if/FS.proto create mode 100644 hbase-native-client/if/Filter.proto create mode 100644 hbase-native-client/if/HBase.proto create mode 100644 hbase-native-client/if/HFile.proto create mode 100644 hbase-native-client/if/LoadBalancer.proto create mode 100644 hbase-native-client/if/MapReduce.proto create mode 100644 hbase-native-client/if/Master.proto create mode 100644 hbase-native-client/if/MasterProcedure.proto create mode 100644 hbase-native-client/if/MultiRowMutation.proto create mode 100644 hbase-native-client/if/Procedure.proto create mode 100644 hbase-native-client/if/Quota.proto create mode 100644 hbase-native-client/if/RPC.proto create mode 100644 hbase-native-client/if/RegionNormalizer.proto create mode 100644 hbase-native-client/if/RegionServerStatus.proto create mode 100644 hbase-native-client/if/RowProcessor.proto create mode 100644 hbase-native-client/if/SecureBulkLoad.proto create mode 100644 hbase-native-client/if/Snapshot.proto create mode 100644 hbase-native-client/if/Tracing.proto create mode 100644 hbase-native-client/if/VisibilityLabels.proto create mode 100644 hbase-native-client/if/WAL.proto create mode 100644 hbase-native-client/if/ZooKeeper.proto delete mode 100644 hbase-native-client/rpc/CMakeLists.txt diff --git a/hbase-native-client/core/BUCK b/hbase-native-client/core/BUCK index a7eaa9e..d6fd8d8 100644 --- a/hbase-native-client/core/BUCK +++ b/hbase-native-client/core/BUCK @@ -41,6 +41,7 @@ cxx_binary( "scanner.cc", ], deps = [ + "//if:if", "//third-party:folly", "//third-party:wangle", ], diff --git a/hbase-native-client/core/client.cc b/hbase-native-client/core/client.cc index 98cf38a..a04daee 100644 --- a/hbase-native-client/core/client.cc +++ b/hbase-native-client/core/client.cc @@ -24,9 +24,13 @@ #include #include +#include "if/ZooKeeper.pb.h" + using namespace folly; +using namespace hbase::pb; int main(int argc, char *argv[]) { + MetaRegionServer mrs; google::ParseCommandLineFlags(&argc, &argv, true); google::InitGoogleLogging(argv[0]); diff --git a/hbase-native-client/core/client.h b/hbase-native-client/core/client.h index 921cecc..35a3bd8 100644 --- a/hbase-native-client/core/client.h +++ b/hbase-native-client/core/client.h @@ -21,4 +21,6 @@ #include +#include "if/Cell.pb.h" + class Client {}; diff --git a/hbase-native-client/if/AccessControl.proto b/hbase-native-client/if/AccessControl.proto new file mode 100644 index 0000000..e67540b --- /dev/null +++ b/hbase-native-client/if/AccessControl.proto @@ -0,0 +1,123 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package hbase.pb; + +option java_package = "org.apache.hadoop.hbase.protobuf.generated"; +option java_outer_classname = "AccessControlProtos"; +option java_generic_services = true; +option java_generate_equals_and_hash = true; +option optimize_for = SPEED; + +import "HBase.proto"; + +message Permission { + enum Action { + READ = 0; + WRITE = 1; + EXEC = 2; + CREATE = 3; + ADMIN = 4; + } + enum Type { + Global = 1; + Namespace = 2; + Table = 3; + } + required Type type = 1; + optional GlobalPermission global_permission = 2; + optional NamespacePermission namespace_permission = 3; + optional TablePermission table_permission = 4; +} + +message TablePermission { + optional TableName table_name = 1; + optional bytes family = 2; + optional bytes qualifier = 3; + repeated Permission.Action action = 4; +} + +message NamespacePermission { + optional bytes namespace_name = 1; + repeated Permission.Action action = 2; +} + +message GlobalPermission { + repeated Permission.Action action = 1; +} + +message UserPermission { + required bytes user = 1; + required Permission permission = 3; +} + +/** + * Content of the /hbase/acl/ znode. + */ +message UsersAndPermissions { + message UserPermissions { + required bytes user = 1; + repeated Permission permissions = 2; + } + + repeated UserPermissions user_permissions = 1; +} + +message GrantRequest { + required UserPermission user_permission = 1; +} + +message GrantResponse { +} + +message RevokeRequest { + required UserPermission user_permission = 1; +} + +message RevokeResponse { +} + +message GetUserPermissionsRequest { + optional Permission.Type type = 1; + optional TableName table_name = 2; + optional bytes namespace_name = 3; +} + +message GetUserPermissionsResponse { + repeated UserPermission user_permission = 1; +} + +message CheckPermissionsRequest { + repeated Permission permission = 1; +} + +message CheckPermissionsResponse { +} + +service AccessControlService { + rpc Grant(GrantRequest) + returns (GrantResponse); + + rpc Revoke(RevokeRequest) + returns (RevokeResponse); + + rpc GetUserPermissions(GetUserPermissionsRequest) + returns (GetUserPermissionsResponse); + + rpc CheckPermissions(CheckPermissionsRequest) + returns (CheckPermissionsResponse); +} diff --git a/hbase-native-client/if/Admin.proto b/hbase-native-client/if/Admin.proto new file mode 100644 index 0000000..e905340 --- /dev/null +++ b/hbase-native-client/if/Admin.proto @@ -0,0 +1,309 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file contains protocol buffers that are used for Admin service. +package hbase.pb; + +option java_package = "org.apache.hadoop.hbase.protobuf.generated"; +option java_outer_classname = "AdminProtos"; +option java_generic_services = true; +option java_generate_equals_and_hash = true; +option optimize_for = SPEED; + +import "HBase.proto"; +import "WAL.proto"; + +message GetRegionInfoRequest { + required RegionSpecifier region = 1; + optional bool compaction_state = 2; +} + +message GetRegionInfoResponse { + required RegionInfo region_info = 1; + optional CompactionState compaction_state = 2; + optional bool isRecovering = 3; + + enum CompactionState { + NONE = 0; + MINOR = 1; + MAJOR = 2; + MAJOR_AND_MINOR = 3; + } +} + +/** + * Get a list of store files for a set of column families in a particular region. + * If no column family is specified, get the store files for all column families. + */ +message GetStoreFileRequest { + required RegionSpecifier region = 1; + repeated bytes family = 2; +} + +message GetStoreFileResponse { + repeated string store_file = 1; +} + +message GetOnlineRegionRequest { +} + +message GetOnlineRegionResponse { + repeated RegionInfo region_info = 1; +} + +message OpenRegionRequest { + repeated RegionOpenInfo open_info = 1; + // the intended server for this RPC. + optional uint64 serverStartCode = 2; + // wall clock time from master + optional uint64 master_system_time = 5; + + message RegionOpenInfo { + required RegionInfo region = 1; + optional uint32 version_of_offline_node = 2; + repeated ServerName favored_nodes = 3; + // open region for distributedLogReplay + optional bool openForDistributedLogReplay = 4; + } +} + +message OpenRegionResponse { + repeated RegionOpeningState opening_state = 1; + + enum RegionOpeningState { + OPENED = 0; + ALREADY_OPENED = 1; + FAILED_OPENING = 2; + } +} + +message WarmupRegionRequest { + + required RegionInfo regionInfo = 1; +} + +message WarmupRegionResponse { +} + +/** + * Closes the specified region and will use or not use ZK during the close + * according to the specified flag. + */ +message CloseRegionRequest { + required RegionSpecifier region = 1; + optional uint32 version_of_closing_node = 2; + optional bool transition_in_ZK = 3 [default = true]; + optional ServerName destination_server = 4; + // the intended server for this RPC. + optional uint64 serverStartCode = 5; +} + +message CloseRegionResponse { + required bool closed = 1; +} + +/** + * Flushes the MemStore of the specified region. + *

+ * This method is synchronous. + */ +message FlushRegionRequest { + required RegionSpecifier region = 1; + optional uint64 if_older_than_ts = 2; + optional bool write_flush_wal_marker = 3; // whether to write a marker to WAL even if not flushed +} + +message FlushRegionResponse { + required uint64 last_flush_time = 1; + optional bool flushed = 2; + optional bool wrote_flush_wal_marker = 3; +} + +/** + * Splits the specified region. + *

+ * This method currently flushes the region and then forces a compaction which + * will then trigger a split. The flush is done synchronously but the + * compaction is asynchronous. + */ +message SplitRegionRequest { + required RegionSpecifier region = 1; + optional bytes split_point = 2; +} + +message SplitRegionResponse { +} + +/** + * Compacts the specified region. Performs a major compaction if specified. + *

+ * This method is asynchronous. + */ +message CompactRegionRequest { + required RegionSpecifier region = 1; + optional bool major = 2; + optional bytes family = 3; +} + +message CompactRegionResponse { +} + +message UpdateFavoredNodesRequest { + repeated RegionUpdateInfo update_info = 1; + + message RegionUpdateInfo { + required RegionInfo region = 1; + repeated ServerName favored_nodes = 2; + } +} + +message UpdateFavoredNodesResponse { + optional uint32 response = 1; +} + +/** + * Merges the specified regions. + *

+ * This method currently closes the regions and then merges them + */ +message MergeRegionsRequest { + required RegionSpecifier region_a = 1; + required RegionSpecifier region_b = 2; + optional bool forcible = 3 [default = false]; + // wall clock time from master + optional uint64 master_system_time = 4; +} + +message MergeRegionsResponse { +} + +// Protocol buffer version of WAL for replication +message WALEntry { + required WALKey key = 1; + // Following may be null if the KVs/Cells are carried along the side in a cellblock (See + // RPC for more on cellblocks). If Cells/KVs are in a cellblock, this next field is null + // and associated_cell_count has count of Cells associated w/ this WALEntry + repeated bytes key_value_bytes = 2; + // If Cell data is carried alongside in a cellblock, this is count of Cells in the cellblock. + optional int32 associated_cell_count = 3; +} + +/** + * Replicates the given entries. The guarantee is that the given entries + * will be durable on the slave cluster if this method returns without + * any exception. hbase.replication has to be set to true for this to work. + */ +message ReplicateWALEntryRequest { + repeated WALEntry entry = 1; + optional string replicationClusterId = 2; + optional string sourceBaseNamespaceDirPath = 3; + optional string sourceHFileArchiveDirPath = 4; +} + +message ReplicateWALEntryResponse { +} + +message RollWALWriterRequest { +} + +/* + * Roll request responses no longer include regions to flush + * this list will always be empty when talking to a 1.0 server + */ +message RollWALWriterResponse { + // A list of encoded name of regions to flush + repeated bytes region_to_flush = 1; +} + +message StopServerRequest { + required string reason = 1; +} + +message StopServerResponse { +} + +message GetServerInfoRequest { +} + +message ServerInfo { + required ServerName server_name = 1; + optional uint32 webui_port = 2; +} + +message GetServerInfoResponse { + required ServerInfo server_info = 1; +} + +message UpdateConfigurationRequest { +} + +message UpdateConfigurationResponse { +} + +service AdminService { + rpc GetRegionInfo(GetRegionInfoRequest) + returns(GetRegionInfoResponse); + + rpc GetStoreFile(GetStoreFileRequest) + returns(GetStoreFileResponse); + + rpc GetOnlineRegion(GetOnlineRegionRequest) + returns(GetOnlineRegionResponse); + + rpc OpenRegion(OpenRegionRequest) + returns(OpenRegionResponse); + + rpc WarmupRegion(WarmupRegionRequest) + returns(WarmupRegionResponse); + + rpc CloseRegion(CloseRegionRequest) + returns(CloseRegionResponse); + + rpc FlushRegion(FlushRegionRequest) + returns(FlushRegionResponse); + + rpc SplitRegion(SplitRegionRequest) + returns(SplitRegionResponse); + + rpc CompactRegion(CompactRegionRequest) + returns(CompactRegionResponse); + + rpc MergeRegions(MergeRegionsRequest) + returns(MergeRegionsResponse); + + rpc ReplicateWALEntry(ReplicateWALEntryRequest) + returns(ReplicateWALEntryResponse); + + rpc Replay(ReplicateWALEntryRequest) + returns(ReplicateWALEntryResponse); + + rpc RollWALWriter(RollWALWriterRequest) + returns(RollWALWriterResponse); + + rpc GetServerInfo(GetServerInfoRequest) + returns(GetServerInfoResponse); + + rpc StopServer(StopServerRequest) + returns(StopServerResponse); + + rpc UpdateFavoredNodes(UpdateFavoredNodesRequest) + returns(UpdateFavoredNodesResponse); + + rpc UpdateConfiguration(UpdateConfigurationRequest) + returns(UpdateConfigurationResponse); +} diff --git a/hbase-native-client/if/Aggregate.proto b/hbase-native-client/if/Aggregate.proto new file mode 100644 index 0000000..4d32e70 --- /dev/null +++ b/hbase-native-client/if/Aggregate.proto @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package hbase.pb; + +option java_package = "org.apache.hadoop.hbase.protobuf.generated"; +option java_outer_classname = "AggregateProtos"; +option java_generic_services = true; +option java_generate_equals_and_hash = true; +option optimize_for = SPEED; + +import "Client.proto"; + +message AggregateRequest { + /** The request passed to the AggregateService consists of three parts + * (1) the (canonical) classname of the ColumnInterpreter implementation + * (2) the Scan query + * (3) any bytes required to construct the ColumnInterpreter object + * properly + */ + required string interpreter_class_name = 1; + required Scan scan = 2; + optional bytes interpreter_specific_bytes = 3; +} + +message AggregateResponse { + /** + * The AggregateService methods all have a response that either is a Pair + * or a simple object. When it is a Pair both first_part and second_part + * have defined values (and the second_part is not present in the response + * when the response is not a pair). Refer to the AggregateImplementation + * class for an overview of the AggregateResponse object constructions. + */ + repeated bytes first_part = 1; + optional bytes second_part = 2; +} + +/** Refer to the AggregateImplementation class for an overview of the + * AggregateService method implementations and their functionality. + */ +service AggregateService { + rpc GetMax (AggregateRequest) returns (AggregateResponse); + rpc GetMin (AggregateRequest) returns (AggregateResponse); + rpc GetSum (AggregateRequest) returns (AggregateResponse); + rpc GetRowNum (AggregateRequest) returns (AggregateResponse); + rpc GetAvg (AggregateRequest) returns (AggregateResponse); + rpc GetStd (AggregateRequest) returns (AggregateResponse); + rpc GetMedian (AggregateRequest) returns (AggregateResponse); +} diff --git a/hbase-native-client/if/Authentication.proto b/hbase-native-client/if/Authentication.proto new file mode 100644 index 0000000..2f64799 --- /dev/null +++ b/hbase-native-client/if/Authentication.proto @@ -0,0 +1,82 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package hbase.pb; + +option java_package = "org.apache.hadoop.hbase.protobuf.generated"; +option java_outer_classname = "AuthenticationProtos"; +option java_generic_services = true; +option java_generate_equals_and_hash = true; +option optimize_for = SPEED; + +message AuthenticationKey { + required int32 id = 1; + required int64 expiration_date = 2; + required bytes key = 3; +} + + +message TokenIdentifier { + enum Kind { + HBASE_AUTH_TOKEN = 0; + } + required Kind kind = 1; + required bytes username = 2; + required int32 key_id = 3; + optional int64 issue_date = 4; + optional int64 expiration_date = 5; + optional int64 sequence_number = 6; +} + + +// Serialization of the org.apache.hadoop.security.token.Token class +// Note that this is a Hadoop class, so fields may change! +message Token { + // the TokenIdentifier in serialized form + // Note: we can't use the protobuf directly because the Hadoop Token class + // only stores the serialized bytes + optional bytes identifier = 1; + optional bytes password = 2; + optional bytes service = 3; +} + + +// RPC request & response messages +message GetAuthenticationTokenRequest { +} + +message GetAuthenticationTokenResponse { + optional Token token = 1; +} + +message WhoAmIRequest { +} + +message WhoAmIResponse { + optional string username = 1; + optional string auth_method = 2; +} + + +// RPC service +service AuthenticationService { + rpc GetAuthenticationToken(GetAuthenticationTokenRequest) + returns (GetAuthenticationTokenResponse); + + rpc WhoAmI(WhoAmIRequest) + returns (WhoAmIResponse); +} diff --git a/hbase-native-client/if/BUCK b/hbase-native-client/if/BUCK new file mode 100644 index 0000000..3490a05 --- /dev/null +++ b/hbase-native-client/if/BUCK @@ -0,0 +1,36 @@ +PROTO_SRCS = glob(['*.proto']) +HEADER_FILENAMES = [ x.replace('.proto','.pb.h') for x in PROTO_SRCS] +CC_FILENAMES = [ x.replace('.proto', '.pb.cc') for x in PROTO_SRCS] + +genrule( + name = 'generate-proto-sources', + srcs = PROTO_SRCS, + cmd = 'mkdir -p $OUT && pwd && protoc --proto_path=. --cpp_out=$OUT *.proto', + out = 'output', +) + +for header_filename in HEADER_FILENAMES: + genrule( + name = header_filename, + cmd = 'mkdir -p `dirname $OUT` ' + ' && cp $(location :generate-proto-sources)/{} $OUT'.format(header_filename), + out = header_filename, + ) +for cc_filename in CC_FILENAMES: + genrule( + name = cc_filename, + cmd = 'mkdir -p `dirname $OUT` ' + ' && cp $(location :generate-proto-sources)/{} $OUT ' + ' && cp $(location :generate-proto-sources)/*.h `dirname $OUT`'.format(cc_filename), + out = cc_filename, + ) + +cxx_library( + name = 'if', + exported_headers = [':' + x for x in HEADER_FILENAMES], + srcs = [':' + x for x in CC_FILENAMES], + deps = [ '//third-party:protobuf'] + + [':' + x for x in CC_FILENAMES] + + [ ':' + x for x in HEADER_FILENAMES ], + visibility = [ 'PUBLIC', ], +) diff --git a/hbase-native-client/if/Cell.proto b/hbase-native-client/if/Cell.proto new file mode 100644 index 0000000..2c61035 --- /dev/null +++ b/hbase-native-client/if/Cell.proto @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Cell and KeyValue protos +package hbase.pb; + +option java_package = "org.apache.hadoop.hbase.protobuf.generated"; +option java_outer_classname = "CellProtos"; +option java_generate_equals_and_hash = true; +option optimize_for = SPEED; + +/** + * The type of the key in a Cell + */ +enum CellType { + MINIMUM = 0; + PUT = 4; + + DELETE = 8; + DELETE_COLUMN = 12; + DELETE_FAMILY = 14; + + // MAXIMUM is used when searching; you look from maximum on down. + MAXIMUM = 255; +} + +/** + * Protocol buffer version of Cell. + */ +message Cell { + optional bytes row = 1; + optional bytes family = 2; + optional bytes qualifier = 3; + optional uint64 timestamp = 4; + optional CellType cell_type = 5; + optional bytes value = 6; + optional bytes tags = 7; +} + +/** + * Protocol buffer version of KeyValue. + * It doesn't have those transient parameters + */ +message KeyValue { + required bytes row = 1; + required bytes family = 2; + required bytes qualifier = 3; + optional uint64 timestamp = 4; + optional CellType key_type = 5; + optional bytes value = 6; + optional bytes tags = 7; +} diff --git a/hbase-native-client/if/Client.proto b/hbase-native-client/if/Client.proto new file mode 100644 index 0000000..ca9abdc --- /dev/null +++ b/hbase-native-client/if/Client.proto @@ -0,0 +1,472 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file contains protocol buffers that are used for Client service. +package hbase.pb; + +option java_package = "org.apache.hadoop.hbase.protobuf.generated"; +option java_outer_classname = "ClientProtos"; +option java_generic_services = true; +option java_generate_equals_and_hash = true; +option optimize_for = SPEED; + +import "HBase.proto"; +import "Filter.proto"; +import "Cell.proto"; +import "Comparator.proto"; +import "MapReduce.proto"; + +/** + * The protocol buffer version of Authorizations. + */ +message Authorizations { + repeated string label = 1; +} + +/** + * The protocol buffer version of CellVisibility. + */ +message CellVisibility { + required string expression = 1; +} + +/** + * Container for a list of column qualifier names of a family. + */ +message Column { + required bytes family = 1; + repeated bytes qualifier = 2; +} + +/** + * Consistency defines the expected consistency level for an operation. + */ +enum Consistency { + STRONG = 0; + TIMELINE = 1; +} + +/** + * The protocol buffer version of Get. + * Unless existence_only is specified, return all the requested data + * for the row that matches exactly. + */ +message Get { + required bytes row = 1; + repeated Column column = 2; + repeated NameBytesPair attribute = 3; + optional Filter filter = 4; + optional TimeRange time_range = 5; + optional uint32 max_versions = 6 [default = 1]; + optional bool cache_blocks = 7 [default = true]; + optional uint32 store_limit = 8; + optional uint32 store_offset = 9; + + // The result isn't asked for, just check for + // the existence. + optional bool existence_only = 10 [default = false]; + + optional Consistency consistency = 12 [default = STRONG]; + repeated ColumnFamilyTimeRange cf_time_range = 13; +} + +message Result { + // Result includes the Cells or else it just has a count of Cells + // that are carried otherwise. + repeated Cell cell = 1; + // The below count is set when the associated cells are + // not part of this protobuf message; they are passed alongside + // and then this Message is just a placeholder with metadata. + // The count is needed to know how many to peel off the block of Cells as + // ours. NOTE: This is different from the pb managed cell_count of the + // 'cell' field above which is non-null when the cells are pb'd. + optional int32 associated_cell_count = 2; + + // used for Get to check existence only. Not set if existence_only was not set to true + // in the query. + optional bool exists = 3; + + // Whether or not the results are coming from possibly stale data + optional bool stale = 4 [default = false]; + + // Whether or not the entire result could be returned. Results will be split when + // the RPC chunk size limit is reached. Partial results contain only a subset of the + // cells for a row and must be combined with a result containing the remaining cells + // to form a complete result + optional bool partial = 5 [default = false]; +} + +/** + * The get request. Perform a single Get operation. + */ +message GetRequest { + required RegionSpecifier region = 1; + required Get get = 2; +} + +message GetResponse { + optional Result result = 1; +} + +/** + * Condition to check if the value of a given cell (row, + * family, qualifier) matches a value via a given comparator. + * + * Condition is used in check and mutate operations. + */ +message Condition { + required bytes row = 1; + required bytes family = 2; + required bytes qualifier = 3; + required CompareType compare_type = 4; + required Comparator comparator = 5; +} + + +/** + * A specific mutation inside a mutate request. + * It can be an append, increment, put or delete based + * on the mutation type. It can be fully filled in or + * only metadata present because data is being carried + * elsewhere outside of pb. + */ +message MutationProto { + optional bytes row = 1; + optional MutationType mutate_type = 2; + repeated ColumnValue column_value = 3; + optional uint64 timestamp = 4; + repeated NameBytesPair attribute = 5; + optional Durability durability = 6 [default = USE_DEFAULT]; + + // For some mutations, a result may be returned, in which case, + // time range can be specified for potential performance gain + optional TimeRange time_range = 7; + // The below count is set when the associated cells are NOT + // part of this protobuf message; they are passed alongside + // and then this Message is a placeholder with metadata. The + // count is needed to know how many to peel off the block of Cells as + // ours. NOTE: This is different from the pb managed cell_count of the + // 'cell' field above which is non-null when the cells are pb'd. + optional int32 associated_cell_count = 8; + + optional uint64 nonce = 9; + + enum Durability { + USE_DEFAULT = 0; + SKIP_WAL = 1; + ASYNC_WAL = 2; + SYNC_WAL = 3; + FSYNC_WAL = 4; + } + + enum MutationType { + APPEND = 0; + INCREMENT = 1; + PUT = 2; + DELETE = 3; + } + + enum DeleteType { + DELETE_ONE_VERSION = 0; + DELETE_MULTIPLE_VERSIONS = 1; + DELETE_FAMILY = 2; + DELETE_FAMILY_VERSION = 3; + } + + message ColumnValue { + required bytes family = 1; + repeated QualifierValue qualifier_value = 2; + + message QualifierValue { + optional bytes qualifier = 1; + optional bytes value = 2; + optional uint64 timestamp = 3; + optional DeleteType delete_type = 4; + optional bytes tags = 5; + } + } +} + +/** + * The mutate request. Perform a single Mutate operation. + * + * Optionally, you can specify a condition. The mutate + * will take place only if the condition is met. Otherwise, + * the mutate will be ignored. In the response result, + * parameter processed is used to indicate if the mutate + * actually happened. + */ +message MutateRequest { + required RegionSpecifier region = 1; + required MutationProto mutation = 2; + optional Condition condition = 3; + optional uint64 nonce_group = 4; +} + +message MutateResponse { + optional Result result = 1; + + // used for mutate to indicate processed only + optional bool processed = 2; +} + +/** + * Instead of get from a table, you can scan it with optional filters. + * You can specify the row key range, time range, the columns/families + * to scan and so on. + * + * This scan is used the first time in a scan request. The response of + * the initial scan will return a scanner id, which should be used to + * fetch result batches later on before it is closed. + */ +message Scan { + repeated Column column = 1; + repeated NameBytesPair attribute = 2; + optional bytes start_row = 3; + optional bytes stop_row = 4; + optional Filter filter = 5; + optional TimeRange time_range = 6; + optional uint32 max_versions = 7 [default = 1]; + optional bool cache_blocks = 8 [default = true]; + optional uint32 batch_size = 9; + optional uint64 max_result_size = 10; + optional uint32 store_limit = 11; + optional uint32 store_offset = 12; + optional bool load_column_families_on_demand = 13; /* DO NOT add defaults to load_column_families_on_demand. */ + optional bool small = 14; + optional bool reversed = 15 [default = false]; + optional Consistency consistency = 16 [default = STRONG]; + optional uint32 caching = 17; + optional bool allow_partial_results = 18; + repeated ColumnFamilyTimeRange cf_time_range = 19; +} + +/** + * A scan request. Initially, it should specify a scan. Later on, you + * can use the scanner id returned to fetch result batches with a different + * scan request. + * + * The scanner will remain open if there are more results, and it's not + * asked to be closed explicitly. + * + * You can fetch the results and ask the scanner to be closed to save + * a trip if you are not interested in remaining results. + */ +message ScanRequest { + optional RegionSpecifier region = 1; + optional Scan scan = 2; + optional uint64 scanner_id = 3; + optional uint32 number_of_rows = 4; + optional bool close_scanner = 5; + optional uint64 next_call_seq = 6; + optional bool client_handles_partials = 7; + optional bool client_handles_heartbeats = 8; + optional bool track_scan_metrics = 9; + optional bool renew = 10 [default = false]; +} + +/** + * The scan response. If there are no more results, more_results will + * be false. If it is not specified, it means there are more. + */ +message ScanResponse { + // This field is filled in if we are doing cellblocks. A cellblock is made up + // of all Cells serialized out as one cellblock BUT responses from a server + // have their Cells grouped by Result. So we can reconstitute the + // Results on the client-side, this field is a list of counts of Cells + // in each Result that makes up the response. For example, if this field + // has 3, 3, 3 in it, then we know that on the client, we are to make + // three Results each of three Cells each. + repeated uint32 cells_per_result = 1; + + optional uint64 scanner_id = 2; + optional bool more_results = 3; + optional uint32 ttl = 4; + // If cells are not carried in an accompanying cellblock, then they are pb'd here. + // This field is mutually exclusive with cells_per_result (since the Cells will + // be inside the pb'd Result) + repeated Result results = 5; + optional bool stale = 6; + + // This field is filled in if we are doing cellblocks. In the event that a row + // could not fit all of its cells into a single RPC chunk, the results will be + // returned as partials, and reconstructed into a complete result on the client + // side. This field is a list of flags indicating whether or not the result + // that the cells belong to is a partial result. For example, if this field + // has false, false, true in it, then we know that on the client side, we need to + // make another RPC request since the last result was only a partial. + repeated bool partial_flag_per_result = 7; + + // A server may choose to limit the number of results returned to the client for + // reasons such as the size in bytes or quantity of results accumulated. This field + // will true when more results exist in the current region. + optional bool more_results_in_region = 8; + + // This field is filled in if the server is sending back a heartbeat message. + // Heartbeat messages are sent back to the client to prevent the scanner from + // timing out. Seeing a heartbeat message communicates to the Client that the + // server would have continued to scan had the time limit not been reached. + optional bool heartbeat_message = 9; + + // This field is filled in if the client has requested that scan metrics be tracked. + // The metrics tracked here are sent back to the client to be tracked together with + // the existing client side metrics. + optional ScanMetrics scan_metrics = 10; +} + +/** + * Atomically bulk load multiple HFiles (say from different column families) + * into an open region. + */ +message BulkLoadHFileRequest { + required RegionSpecifier region = 1; + repeated FamilyPath family_path = 2; + optional bool assign_seq_num = 3; + + message FamilyPath { + required bytes family = 1; + required string path = 2; + } +} + +message BulkLoadHFileResponse { + required bool loaded = 1; +} + +message CoprocessorServiceCall { + required bytes row = 1; + required string service_name = 2; + required string method_name = 3; + required bytes request = 4; +} + +message CoprocessorServiceResult { + optional NameBytesPair value = 1; +} + +message CoprocessorServiceRequest { + required RegionSpecifier region = 1; + required CoprocessorServiceCall call = 2; +} + +message CoprocessorServiceResponse { + required RegionSpecifier region = 1; + required NameBytesPair value = 2; +} + +// Either a Get or a Mutation +message Action { + // If part of a multi action, useful aligning + // result with what was originally submitted. + optional uint32 index = 1; + optional MutationProto mutation = 2; + optional Get get = 3; + optional CoprocessorServiceCall service_call = 4; +} + +/** + * Actions to run against a Region. + */ +message RegionAction { + required RegionSpecifier region = 1; + // When set, run mutations as atomic unit. + optional bool atomic = 2; + repeated Action action = 3; +} + +/* +* Statistics about the current load on the region +*/ +message RegionLoadStats { + // Percent load on the memstore. Guaranteed to be positive, between 0 and 100. + optional int32 memstoreLoad = 1 [default = 0]; + // Percent JVM heap occupancy. Guaranteed to be positive, between 0 and 100. + // We can move this to "ServerLoadStats" should we develop them. + optional int32 heapOccupancy = 2 [default = 0]; + // Compaction pressure. Guaranteed to be positive, between 0 and 100. + optional int32 compactionPressure = 3 [default = 0]; +} + +/** + * Either a Result or an Exception NameBytesPair (keyed by + * exception name whose value is the exception stringified) + * or maybe empty if no result and no exception. + */ +message ResultOrException { + // If part of a multi call, save original index of the list of all + // passed so can align this response w/ original request. + optional uint32 index = 1; + optional Result result = 2; + optional NameBytesPair exception = 3; + // result if this was a coprocessor service call + optional CoprocessorServiceResult service_result = 4; + // current load on the region + optional RegionLoadStats loadStats = 5; +} + +/** + * The result of a RegionAction. + */ +message RegionActionResult { + repeated ResultOrException resultOrException = 1; + // If the operation failed globally for this region, this exception is set + optional NameBytesPair exception = 2; +} + +/** + * Execute a list of actions on a given region in order. + * Nothing prevents a request to contains a set of RegionAction on the same region. + * For this reason, the matching between the MultiRequest and the MultiResponse is not + * done by the region specifier but by keeping the order of the RegionActionResult vs. + * the order of the RegionAction. + */ +message MultiRequest { + repeated RegionAction regionAction = 1; + optional uint64 nonceGroup = 2; + optional Condition condition = 3; +} + +message MultiResponse { + repeated RegionActionResult regionActionResult = 1; + // used for mutate to indicate processed only + optional bool processed = 2; +} + + +service ClientService { + rpc Get(GetRequest) + returns(GetResponse); + + rpc Mutate(MutateRequest) + returns(MutateResponse); + + rpc Scan(ScanRequest) + returns(ScanResponse); + + rpc BulkLoadHFile(BulkLoadHFileRequest) + returns(BulkLoadHFileResponse); + + rpc ExecService(CoprocessorServiceRequest) + returns(CoprocessorServiceResponse); + + rpc ExecRegionServerService(CoprocessorServiceRequest) + returns(CoprocessorServiceResponse); + + rpc Multi(MultiRequest) + returns(MultiResponse); +} diff --git a/hbase-native-client/if/ClusterId.proto b/hbase-native-client/if/ClusterId.proto new file mode 100644 index 0000000..aed8cfc --- /dev/null +++ b/hbase-native-client/if/ClusterId.proto @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file contains protocol buffers that are shared throughout HBase +package hbase.pb; + +option java_package = "org.apache.hadoop.hbase.protobuf.generated"; +option java_outer_classname = "ClusterIdProtos"; +option java_generate_equals_and_hash = true; +option optimize_for = SPEED; + +/** + * Content of the '/hbase/hbaseid', cluster id, znode. + * Also cluster of the ${HBASE_ROOTDIR}/hbase.id file. + */ +message ClusterId { + // This is the cluster id, a uuid as a String + required string cluster_id = 1; +} diff --git a/hbase-native-client/if/ClusterStatus.proto b/hbase-native-client/if/ClusterStatus.proto new file mode 100644 index 0000000..228be7e --- /dev/null +++ b/hbase-native-client/if/ClusterStatus.proto @@ -0,0 +1,224 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file contains protocol buffers that are used for ClustStatus +package hbase.pb; + +option java_package = "org.apache.hadoop.hbase.protobuf.generated"; +option java_outer_classname = "ClusterStatusProtos"; +option java_generate_equals_and_hash = true; +option optimize_for = SPEED; + +import "HBase.proto"; +import "ClusterId.proto"; +import "FS.proto"; + +message RegionState { + required RegionInfo region_info = 1; + required State state = 2; + optional uint64 stamp = 3; + enum State { + OFFLINE = 0; // region is in an offline state + PENDING_OPEN = 1; // sent rpc to server to open but has not begun + OPENING = 2; // server has begun to open but not yet done + OPEN = 3; // server opened region and updated meta + PENDING_CLOSE = 4; // sent rpc to server to close but has not begun + CLOSING = 5; // server has begun to close but not yet done + CLOSED = 6; // server closed region and updated meta + SPLITTING = 7; // server started split of a region + SPLIT = 8; // server completed split of a region + FAILED_OPEN = 9; // failed to open, and won't retry any more + FAILED_CLOSE = 10; // failed to close, and won't retry any more + MERGING = 11; // server started merge a region + MERGED = 12; // server completed merge of a region + SPLITTING_NEW = 13; // new region to be created when RS splits a parent + // region but hasn't be created yet, or master doesn't + // know it's already created + MERGING_NEW = 14; // new region to be created when RS merges two + // daughter regions but hasn't be created yet, or + // master doesn't know it's already created + } +} + +message RegionInTransition { + required RegionSpecifier spec = 1; + required RegionState region_state = 2; +} + +/** + * sequence Id of a store + */ +message StoreSequenceId { + required bytes family_name = 1; + required uint64 sequence_id = 2; +} + +/** + * contains a sequence id of a region which should be the minimum of its store sequence ids and + * list of sequence ids of the region's stores + */ +message RegionStoreSequenceIds { + required uint64 last_flushed_sequence_id = 1; + repeated StoreSequenceId store_sequence_id = 2; +} + +message RegionLoad { + /** the region specifier */ + required RegionSpecifier region_specifier = 1; + + /** the number of stores for the region */ + optional uint32 stores = 2; + + /** the number of storefiles for the region */ + optional uint32 storefiles = 3; + + /** the total size of the store files for the region, uncompressed, in MB */ + optional uint32 store_uncompressed_size_MB = 4; + + /** the current total size of the store files for the region, in MB */ + optional uint32 storefile_size_MB = 5; + + /** the current size of the memstore for the region, in MB */ + optional uint32 memstore_size_MB = 6; + + /** + * The current total size of root-level store file indexes for the region, + * in MB. The same as {@link #rootIndexSizeKB} but in MB. + */ + optional uint32 storefile_index_size_MB = 7; + + /** the current total read requests made to region */ + optional uint64 read_requests_count = 8; + + /** the current total write requests made to region */ + optional uint64 write_requests_count = 9; + + /** the total compacting key values in currently running compaction */ + optional uint64 total_compacting_KVs = 10; + + /** the completed count of key values in currently running compaction */ + optional uint64 current_compacted_KVs = 11; + + /** The current total size of root-level indexes for the region, in KB. */ + optional uint32 root_index_size_KB = 12; + + /** The total size of all index blocks, not just the root level, in KB. */ + optional uint32 total_static_index_size_KB = 13; + + /** + * The total size of all Bloom filter blocks, not just loaded into the + * block cache, in KB. + */ + optional uint32 total_static_bloom_size_KB = 14; + + /** the most recent sequence Id from cache flush */ + optional uint64 complete_sequence_id = 15; + + /** The current data locality for region in the regionserver */ + optional float data_locality = 16; + + optional uint64 last_major_compaction_ts = 17 [default = 0]; + + /** the most recent sequence Id of store from cache flush */ + repeated StoreSequenceId store_complete_sequence_id = 18; +} + +/* Server-level protobufs */ + +message ReplicationLoadSink { + required uint64 ageOfLastAppliedOp = 1; + required uint64 timeStampsOfLastAppliedOp = 2; +} + +message ReplicationLoadSource { + required string peerID = 1; + required uint64 ageOfLastShippedOp = 2; + required uint32 sizeOfLogQueue = 3; + required uint64 timeStampOfLastShippedOp = 4; + required uint64 replicationLag = 5; +} + +message ServerLoad { + /** Number of requests since last report. */ + optional uint64 number_of_requests = 1; + + /** Total Number of requests from the start of the region server. */ + optional uint64 total_number_of_requests = 2; + + /** the amount of used heap, in MB. */ + optional uint32 used_heap_MB = 3; + + /** the maximum allowable size of the heap, in MB. */ + optional uint32 max_heap_MB = 4; + + /** Information on the load of individual regions. */ + repeated RegionLoad region_loads = 5; + + /** + * Regionserver-level coprocessors, e.g., WALObserver implementations. + * Region-level coprocessors, on the other hand, are stored inside RegionLoad + * objects. + */ + repeated Coprocessor coprocessors = 6; + + /** + * Time when incremental (non-total) counts began being calculated (e.g. number_of_requests) + * time is measured as the difference, measured in milliseconds, between the current time + * and midnight, January 1, 1970 UTC. + */ + optional uint64 report_start_time = 7; + + /** + * Time when report was generated. + * time is measured as the difference, measured in milliseconds, between the current time + * and midnight, January 1, 1970 UTC. + */ + optional uint64 report_end_time = 8; + + /** + * The port number that this region server is hosing an info server on. + */ + optional uint32 info_server_port = 9; + + /** + * The replicationLoadSource for the replication Source status of this region server. + */ + repeated ReplicationLoadSource replLoadSource = 10; + + /** + * The replicationLoadSink for the replication Sink status of this region server. + */ + optional ReplicationLoadSink replLoadSink = 11; +} + +message LiveServerInfo { + required ServerName server = 1; + required ServerLoad server_load = 2; +} + +message ClusterStatus { + optional HBaseVersionFileContent hbase_version = 1; + repeated LiveServerInfo live_servers = 2; + repeated ServerName dead_servers = 3; + repeated RegionInTransition regions_in_transition = 4; + optional ClusterId cluster_id = 5; + repeated Coprocessor master_coprocessors = 6; + optional ServerName master = 7; + repeated ServerName backup_masters = 8; + optional bool balancer_on = 9; +} diff --git a/hbase-native-client/if/Comparator.proto b/hbase-native-client/if/Comparator.proto new file mode 100644 index 0000000..496b68d --- /dev/null +++ b/hbase-native-client/if/Comparator.proto @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file contains protocol buffers that are used for filters +package hbase.pb; + +option java_package = "org.apache.hadoop.hbase.protobuf.generated"; +option java_outer_classname = "ComparatorProtos"; +option java_generic_services = true; +option java_generate_equals_and_hash = true; +option optimize_for = SPEED; + +// This file contains protocol buffers that are used for comparators (e.g. in filters) + +message Comparator { + required string name = 1; + optional bytes serialized_comparator = 2; +} + +message ByteArrayComparable { + optional bytes value = 1; +} + +message BinaryComparator { + required ByteArrayComparable comparable = 1; +} + +message LongComparator { + required ByteArrayComparable comparable = 1; +} + +message BinaryPrefixComparator { + required ByteArrayComparable comparable = 1; +} + +message BitComparator { + required ByteArrayComparable comparable = 1; + required BitwiseOp bitwise_op = 2; + + enum BitwiseOp { + AND = 1; + OR = 2; + XOR = 3; + } +} + +message NullComparator { +} + +message RegexStringComparator { + required string pattern = 1; + required int32 pattern_flags = 2; + required string charset = 3; + optional string engine = 4; +} + +message SubstringComparator { + required string substr = 1; +} diff --git a/hbase-native-client/if/Encryption.proto b/hbase-native-client/if/Encryption.proto new file mode 100644 index 0000000..97ab5b2 --- /dev/null +++ b/hbase-native-client/if/Encryption.proto @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file contains protocol buffers used for encryption +package hbase.pb; + +option java_package = "org.apache.hadoop.hbase.protobuf.generated"; +option java_outer_classname = "EncryptionProtos"; +option java_generate_equals_and_hash = true; +option optimize_for = SPEED; + +message WrappedKey { + required string algorithm = 1; + required uint32 length = 2; + required bytes data = 3; + optional bytes iv = 4; + optional bytes hash = 5; +} diff --git a/hbase-native-client/if/ErrorHandling.proto b/hbase-native-client/if/ErrorHandling.proto new file mode 100644 index 0000000..be9a743 --- /dev/null +++ b/hbase-native-client/if/ErrorHandling.proto @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file contains protocol buffers that are used for error handling +package hbase.pb; + +option java_package = "org.apache.hadoop.hbase.protobuf.generated"; +option java_outer_classname = "ErrorHandlingProtos"; +option java_generate_equals_and_hash = true; +option optimize_for = SPEED; + +/** + * Protobuf version of a java.lang.StackTraceElement + * so we can serialize exceptions. + */ +message StackTraceElementMessage { + optional string declaring_class = 1; + optional string method_name = 2; + optional string file_name = 3; + optional int32 line_number = 4; +} + +/** + * Cause of a remote failure for a generic exception. Contains + * all the information for a generic exception as well as + * optional info about the error for generic info passing + * (which should be another protobuffed class). + */ +message GenericExceptionMessage { + optional string class_name = 1; + optional string message = 2; + optional bytes error_info = 3; + repeated StackTraceElementMessage trace = 4; +} + +/** + * Exception sent across the wire when a remote task needs + * to notify other tasks that it failed and why + */ +message ForeignExceptionMessage { + optional string source = 1; + optional GenericExceptionMessage generic_exception = 2; +} diff --git a/hbase-native-client/if/FS.proto b/hbase-native-client/if/FS.proto new file mode 100644 index 0000000..9e93120 --- /dev/null +++ b/hbase-native-client/if/FS.proto @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file contains protocol buffers that are written into the filesystem +package hbase.pb; + +option java_package = "org.apache.hadoop.hbase.protobuf.generated"; +option java_outer_classname = "FSProtos"; +option java_generate_equals_and_hash = true; +option optimize_for = SPEED; + +/** + * The ${HBASE_ROOTDIR}/hbase.version file content + */ +message HBaseVersionFileContent { + required string version = 1; +} + +/** + * Reference file content used when we split an hfile under a region. + */ +message Reference { + required bytes splitkey = 1; + enum Range { + TOP = 0; + BOTTOM = 1; + } + required Range range = 2; +} + diff --git a/hbase-native-client/if/Filter.proto b/hbase-native-client/if/Filter.proto new file mode 100644 index 0000000..67d5717 --- /dev/null +++ b/hbase-native-client/if/Filter.proto @@ -0,0 +1,170 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file contains protocol buffers that are used for filters +package hbase.pb; + +option java_package = "org.apache.hadoop.hbase.protobuf.generated"; +option java_outer_classname = "FilterProtos"; +option java_generic_services = true; +option java_generate_equals_and_hash = true; +option optimize_for = SPEED; + +import "HBase.proto"; +import "Comparator.proto"; + +message Filter { + required string name = 1; + optional bytes serialized_filter = 2; +} + +message ColumnCountGetFilter { + required int32 limit = 1; +} + +message ColumnPaginationFilter { + required int32 limit = 1; + optional int32 offset = 2; + optional bytes column_offset = 3; +} + +message ColumnPrefixFilter { + required bytes prefix = 1; +} + +message ColumnRangeFilter { + optional bytes min_column = 1; + optional bool min_column_inclusive = 2; + optional bytes max_column = 3; + optional bool max_column_inclusive = 4; +} + +message CompareFilter { + required CompareType compare_op = 1; + optional Comparator comparator = 2; +} + +message DependentColumnFilter { + required CompareFilter compare_filter = 1; + optional bytes column_family = 2; + optional bytes column_qualifier = 3; + optional bool drop_dependent_column = 4; +} + +message FamilyFilter { + required CompareFilter compare_filter = 1; +} + +message FilterList { + required Operator operator = 1; + repeated Filter filters = 2; + + enum Operator { + MUST_PASS_ALL = 1; + MUST_PASS_ONE = 2; + } +} + +message FilterWrapper { + required Filter filter = 1; +} + +message FirstKeyOnlyFilter { +} + +message FirstKeyValueMatchingQualifiersFilter { + repeated bytes qualifiers = 1; +} + +message FuzzyRowFilter { + repeated BytesBytesPair fuzzy_keys_data = 1; +} + +message InclusiveStopFilter { + optional bytes stop_row_key = 1; +} + +message KeyOnlyFilter { + required bool len_as_val = 1; +} + +message MultipleColumnPrefixFilter { + repeated bytes sorted_prefixes = 1; +} + +message PageFilter { + required int64 page_size = 1; +} + +message PrefixFilter { + optional bytes prefix = 1; +} + +message QualifierFilter { + required CompareFilter compare_filter = 1; +} + +message RandomRowFilter { + required float chance = 1; +} + +message RowFilter { + required CompareFilter compare_filter = 1; +} + +message SingleColumnValueExcludeFilter { + required SingleColumnValueFilter single_column_value_filter = 1; +} + +message SingleColumnValueFilter { + optional bytes column_family = 1; + optional bytes column_qualifier = 2; + required CompareType compare_op = 3; + required Comparator comparator = 4; + optional bool filter_if_missing = 5; + optional bool latest_version_only = 6; +} + +message SkipFilter { + required Filter filter = 1; +} + +message TimestampsFilter { + repeated int64 timestamps = 1 [packed=true]; +} + +message ValueFilter { + required CompareFilter compare_filter = 1; +} + +message WhileMatchFilter { + required Filter filter = 1; +} +message FilterAllFilter { +} + +message RowRange { + optional bytes start_row = 1; + optional bool start_row_inclusive = 2; + optional bytes stop_row = 3; + optional bool stop_row_inclusive =4; +} + +message MultiRowRangeFilter { + repeated RowRange row_range_list = 1; +} \ No newline at end of file diff --git a/hbase-native-client/if/HBase.proto b/hbase-native-client/if/HBase.proto new file mode 100644 index 0000000..e5c967a --- /dev/null +++ b/hbase-native-client/if/HBase.proto @@ -0,0 +1,258 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file contains protocol buffers that are shared throughout HBase +package hbase.pb; + +option java_package = "org.apache.hadoop.hbase.protobuf.generated"; +option java_outer_classname = "HBaseProtos"; +option java_generate_equals_and_hash = true; +option optimize_for = SPEED; + +import "Cell.proto"; + +/** + * Table Name + */ +message TableName { + required bytes namespace = 1; + required bytes qualifier = 2; +} + +/** + * Table Schema + * Inspired by the rest TableSchema + */ +message TableSchema { + optional TableName table_name = 1; + repeated BytesBytesPair attributes = 2; + repeated ColumnFamilySchema column_families = 3; + repeated NameStringPair configuration = 4; +} + +/** Denotes state of the table */ +message TableState { + // Table's current state + enum State { + ENABLED = 0; + DISABLED = 1; + DISABLING = 2; + ENABLING = 3; + } + // This is the table's state. + required State state = 1; +} + +/** On HDFS representation of table state. */ +message TableDescriptor { + required TableSchema schema = 1; +} + +/** + * Column Family Schema + * Inspired by the rest ColumSchemaMessage + */ +message ColumnFamilySchema { + required bytes name = 1; + repeated BytesBytesPair attributes = 2; + repeated NameStringPair configuration = 3; +} + +/** + * Protocol buffer version of HRegionInfo. + */ +message RegionInfo { + required uint64 region_id = 1; + required TableName table_name = 2; + optional bytes start_key = 3; + optional bytes end_key = 4; + optional bool offline = 5; + optional bool split = 6; + optional int32 replica_id = 7 [default = 0]; +} + +/** + * Protocol buffer for favored nodes + */ +message FavoredNodes { + repeated ServerName favored_node = 1; +} + +/** + * Container protocol buffer to specify a region. + * You can specify region by region name, or the hash + * of the region name, which is known as encoded + * region name. + */ +message RegionSpecifier { + required RegionSpecifierType type = 1; + required bytes value = 2; + + enum RegionSpecifierType { + // ,,. + REGION_NAME = 1; + + // hash of ,, + ENCODED_REGION_NAME = 2; + } +} + +/** + * A range of time. Both from and to are Java time + * stamp in milliseconds. If you don't specify a time + * range, it means all time. By default, if not + * specified, from = 0, and to = Long.MAX_VALUE + */ +message TimeRange { + optional uint64 from = 1; + optional uint64 to = 2; +} + +/* ColumnFamily Specific TimeRange */ +message ColumnFamilyTimeRange { + required bytes column_family = 1; + required TimeRange time_range = 2; +} + +/* Comparison operators */ +enum CompareType { + LESS = 0; + LESS_OR_EQUAL = 1; + EQUAL = 2; + NOT_EQUAL = 3; + GREATER_OR_EQUAL = 4; + GREATER = 5; + NO_OP = 6; +} + +/** + * Protocol buffer version of ServerName + */ +message ServerName { + required string host_name = 1; + optional uint32 port = 2; + optional uint64 start_code = 3; +} + +// Comment data structures + +message Coprocessor { + required string name = 1; +} + +message NameStringPair { + required string name = 1; + required string value = 2; +} + +message NameBytesPair { + required string name = 1; + optional bytes value = 2; +} + +message BytesBytesPair { + required bytes first = 1; + required bytes second = 2; +} + +message NameInt64Pair { + optional string name = 1; + optional int64 value = 2; +} + +/** + * Description of the snapshot to take + */ +message SnapshotDescription { + required string name = 1; + optional string table = 2; // not needed for delete, but checked for in taking snapshot + optional int64 creation_time = 3 [default = 0]; + enum Type { + DISABLED = 0; + FLUSH = 1; + SKIPFLUSH = 2; + } + optional Type type = 4 [default = FLUSH]; + optional int32 version = 5; + optional string owner = 6; +} + +/** + * Description of the distributed procedure to take + */ +message ProcedureDescription { + required string signature = 1; // the unique signature of the procedure + optional string instance = 2; // the procedure instance name + optional int64 creation_time = 3 [default = 0]; + repeated NameStringPair configuration = 4; +} + +message EmptyMsg { +} + +enum TimeUnit { + NANOSECONDS = 1; + MICROSECONDS = 2; + MILLISECONDS = 3; + SECONDS = 4; + MINUTES = 5; + HOURS = 6; + DAYS = 7; +} + +message LongMsg { + required int64 long_msg = 1; +} + +message DoubleMsg { + required double double_msg = 1; +} + +message BigDecimalMsg { + required bytes bigdecimal_msg = 1; +} + +message UUID { + required uint64 least_sig_bits = 1; + required uint64 most_sig_bits = 2; +} + +message NamespaceDescriptor { + required bytes name = 1; + repeated NameStringPair configuration = 2; +} + +// Rpc client version info proto. Included in ConnectionHeader on connection setup +message VersionInfo { + required string version = 1; + required string url = 2; + required string revision = 3; + required string user = 4; + required string date = 5; + required string src_checksum = 6; + optional uint32 version_major = 7; + optional uint32 version_minor = 8; +} + +/** + * Description of the region server info + */ +message RegionServerInfo { + optional int32 infoPort = 1; + optional VersionInfo version_info = 2; +} diff --git a/hbase-native-client/if/HFile.proto b/hbase-native-client/if/HFile.proto new file mode 100644 index 0000000..5c5e4f3 --- /dev/null +++ b/hbase-native-client/if/HFile.proto @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package hbase.pb; + +option java_package = "org.apache.hadoop.hbase.protobuf.generated"; +option java_outer_classname = "HFileProtos"; +option java_generic_services = true; +option java_generate_equals_and_hash = true; +option optimize_for = SPEED; + +import "HBase.proto"; + +// Map of name/values +message FileInfoProto { + repeated BytesBytesPair map_entry = 1; +} + +// HFile file trailer +message FileTrailerProto { + optional uint64 file_info_offset = 1; + optional uint64 load_on_open_data_offset = 2; + optional uint64 uncompressed_data_index_size = 3; + optional uint64 total_uncompressed_bytes = 4; + optional uint32 data_index_count = 5; + optional uint32 meta_index_count = 6; + optional uint64 entry_count = 7; + optional uint32 num_data_index_levels = 8; + optional uint64 first_data_block_offset = 9; + optional uint64 last_data_block_offset = 10; + optional string comparator_class_name = 11; + optional uint32 compression_codec = 12; + optional bytes encryption_key = 13; +} diff --git a/hbase-native-client/if/LoadBalancer.proto b/hbase-native-client/if/LoadBalancer.proto new file mode 100644 index 0000000..f9c5d0d --- /dev/null +++ b/hbase-native-client/if/LoadBalancer.proto @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file contains protocol buffers to represent the state of the load balancer. +package hbase.pb; + +option java_package = "org.apache.hadoop.hbase.protobuf.generated"; +option java_outer_classname = "LoadBalancerProtos"; +option java_generate_equals_and_hash = true; +option optimize_for = SPEED; + +message LoadBalancerState { + optional bool balancer_on = 1; +} diff --git a/hbase-native-client/if/MapReduce.proto b/hbase-native-client/if/MapReduce.proto new file mode 100644 index 0000000..f96ffdf --- /dev/null +++ b/hbase-native-client/if/MapReduce.proto @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + //This file includes protocol buffers used in MapReduce only. +package hbase.pb; + +option java_package = "org.apache.hadoop.hbase.protobuf.generated"; +option java_outer_classname = "MapReduceProtos"; +option java_generate_equals_and_hash = true; +option optimize_for = SPEED; + +import "HBase.proto"; + +message ScanMetrics { + repeated NameInt64Pair metrics = 1; +} + +message TableSnapshotRegionSplit { + repeated string locations = 2; + optional TableSchema table = 3; + optional RegionInfo region = 4; +} diff --git a/hbase-native-client/if/Master.proto b/hbase-native-client/if/Master.proto new file mode 100644 index 0000000..4d3a2e1 --- /dev/null +++ b/hbase-native-client/if/Master.proto @@ -0,0 +1,778 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// All to do with the Master. Includes schema management since these +// changes are run by the Master process. +package hbase.pb; + +option java_package = "org.apache.hadoop.hbase.protobuf.generated"; +option java_outer_classname = "MasterProtos"; +option java_generic_services = true; +option java_generate_equals_and_hash = true; +option optimize_for = SPEED; + +import "HBase.proto"; +import "Client.proto"; +import "ClusterStatus.proto"; +import "ErrorHandling.proto"; +import "Procedure.proto"; +import "Quota.proto"; + +/* Column-level protobufs */ + +message AddColumnRequest { + required TableName table_name = 1; + required ColumnFamilySchema column_families = 2; + optional uint64 nonce_group = 3 [default = 0]; + optional uint64 nonce = 4 [default = 0]; +} + +message AddColumnResponse { + optional uint64 proc_id = 1; +} + +message DeleteColumnRequest { + required TableName table_name = 1; + required bytes column_name = 2; + optional uint64 nonce_group = 3 [default = 0]; + optional uint64 nonce = 4 [default = 0]; +} + +message DeleteColumnResponse { + optional uint64 proc_id = 1; +} + +message ModifyColumnRequest { + required TableName table_name = 1; + required ColumnFamilySchema column_families = 2; + optional uint64 nonce_group = 3 [default = 0]; + optional uint64 nonce = 4 [default = 0]; +} + +message ModifyColumnResponse { + optional uint64 proc_id = 1; +} + +/* Region-level Protos */ + +message MoveRegionRequest { + required RegionSpecifier region = 1; + optional ServerName dest_server_name = 2; +} + +message MoveRegionResponse { +} + +/** + * Dispatch merging the specified regions. + */ +message DispatchMergingRegionsRequest { + required RegionSpecifier region_a = 1; + required RegionSpecifier region_b = 2; + optional bool forcible = 3 [default = false]; +} + +message DispatchMergingRegionsResponse { +} + +message AssignRegionRequest { + required RegionSpecifier region = 1; +} + +message AssignRegionResponse { +} + +message UnassignRegionRequest { + required RegionSpecifier region = 1; + optional bool force = 2 [default = false]; +} + +message UnassignRegionResponse { +} + +message OfflineRegionRequest { + required RegionSpecifier region = 1; +} + +message OfflineRegionResponse { +} + +/* Table-level protobufs */ + +message CreateTableRequest { + required TableSchema table_schema = 1; + repeated bytes split_keys = 2; + optional uint64 nonce_group = 3 [default = 0]; + optional uint64 nonce = 4 [default = 0]; +} + +message CreateTableResponse { + optional uint64 proc_id = 1; +} + +message DeleteTableRequest { + required TableName table_name = 1; + optional uint64 nonce_group = 2 [default = 0]; + optional uint64 nonce = 3 [default = 0]; +} + +message DeleteTableResponse { + optional uint64 proc_id = 1; +} + +message TruncateTableRequest { + required TableName tableName = 1; + optional bool preserveSplits = 2 [default = false]; + optional uint64 nonce_group = 3 [default = 0]; + optional uint64 nonce = 4 [default = 0]; +} + +message TruncateTableResponse { + optional uint64 proc_id = 1; +} + +message EnableTableRequest { + required TableName table_name = 1; + optional uint64 nonce_group = 2 [default = 0]; + optional uint64 nonce = 3 [default = 0]; +} + +message EnableTableResponse { + optional uint64 proc_id = 1; +} + +message DisableTableRequest { + required TableName table_name = 1; + optional uint64 nonce_group = 2 [default = 0]; + optional uint64 nonce = 3 [default = 0]; +} + +message DisableTableResponse { + optional uint64 proc_id = 1; +} + +message ModifyTableRequest { + required TableName table_name = 1; + required TableSchema table_schema = 2; + optional uint64 nonce_group = 3 [default = 0]; + optional uint64 nonce = 4 [default = 0]; +} + +message ModifyTableResponse { + optional uint64 proc_id = 1; +} + +/* Namespace-level protobufs */ + +message CreateNamespaceRequest { + required NamespaceDescriptor namespaceDescriptor = 1; + optional uint64 nonce_group = 2 [default = 0]; + optional uint64 nonce = 3 [default = 0]; +} + +message CreateNamespaceResponse { +} + +message DeleteNamespaceRequest { + required string namespaceName = 1; + optional uint64 nonce_group = 2 [default = 0]; + optional uint64 nonce = 3 [default = 0]; +} + +message DeleteNamespaceResponse { +} + +message ModifyNamespaceRequest { + required NamespaceDescriptor namespaceDescriptor = 1; + optional uint64 nonce_group = 2 [default = 0]; + optional uint64 nonce = 3 [default = 0]; +} + +message ModifyNamespaceResponse { +} + +message GetNamespaceDescriptorRequest { + required string namespaceName = 1; +} + +message GetNamespaceDescriptorResponse { + required NamespaceDescriptor namespaceDescriptor = 1; +} + +message ListNamespaceDescriptorsRequest { +} + +message ListNamespaceDescriptorsResponse { + repeated NamespaceDescriptor namespaceDescriptor = 1; +} + +message ListTableDescriptorsByNamespaceRequest { + required string namespaceName = 1; +} + +message ListTableDescriptorsByNamespaceResponse { + repeated TableSchema tableSchema = 1; +} + +message ListTableNamesByNamespaceRequest { + required string namespaceName = 1; +} + +message ListTableNamesByNamespaceResponse { + repeated TableName tableName = 1; +} + +/* Cluster-level protobufs */ + + +message ShutdownRequest { +} + +message ShutdownResponse { +} + +message StopMasterRequest { +} + +message StopMasterResponse { +} + +message BalanceRequest { + optional bool force = 1; +} + +message BalanceResponse { + required bool balancer_ran = 1; +} + +message SetBalancerRunningRequest { + required bool on = 1; + optional bool synchronous = 2; +} + +message SetBalancerRunningResponse { + optional bool prev_balance_value = 1; +} + +message IsBalancerEnabledRequest { +} + +message IsBalancerEnabledResponse { + required bool enabled = 1; +} + +message NormalizeRequest { +} + +message NormalizeResponse { + required bool normalizer_ran = 1; +} + +message SetNormalizerRunningRequest { + required bool on = 1; +} + +message SetNormalizerRunningResponse { + optional bool prev_normalizer_value = 1; +} + +message IsNormalizerEnabledRequest { +} + +message IsNormalizerEnabledResponse { + required bool enabled = 1; +} + +message RunCatalogScanRequest { +} + +message RunCatalogScanResponse { + optional int32 scan_result = 1; +} + +message EnableCatalogJanitorRequest { + required bool enable = 1; +} + +message EnableCatalogJanitorResponse { + optional bool prev_value = 1; +} + +message IsCatalogJanitorEnabledRequest { +} + +message IsCatalogJanitorEnabledResponse { + required bool value = 1; +} + +message SnapshotRequest { + required SnapshotDescription snapshot = 1; +} + +message SnapshotResponse { + required int64 expected_timeout = 1; +} + +message GetCompletedSnapshotsRequest { +} + +message GetCompletedSnapshotsResponse { + repeated SnapshotDescription snapshots = 1; +} + +message DeleteSnapshotRequest { + required SnapshotDescription snapshot = 1; +} + +message DeleteSnapshotResponse { +} + +message RestoreSnapshotRequest { + required SnapshotDescription snapshot = 1; +} + +message RestoreSnapshotResponse { +} + +/* if you don't send the snapshot, then you will get it back + * in the response (if the snapshot is done) so you can check the snapshot + */ +message IsSnapshotDoneRequest { + optional SnapshotDescription snapshot = 1; +} + +message IsSnapshotDoneResponse { + optional bool done = 1 [default = false]; + optional SnapshotDescription snapshot = 2; +} + +message IsRestoreSnapshotDoneRequest { + optional SnapshotDescription snapshot = 1; +} + +message IsRestoreSnapshotDoneResponse { + optional bool done = 1 [default = false]; +} + +message GetSchemaAlterStatusRequest { + required TableName table_name = 1; +} + +message GetSchemaAlterStatusResponse { + optional uint32 yet_to_update_regions = 1; + optional uint32 total_regions = 2; +} + +message GetTableDescriptorsRequest { + repeated TableName table_names = 1; + optional string regex = 2; + optional bool include_sys_tables = 3 [default=false]; + optional string namespace = 4; +} + +message GetTableDescriptorsResponse { + repeated TableSchema table_schema = 1; +} + +message GetTableNamesRequest { + optional string regex = 1; + optional bool include_sys_tables = 2 [default=false]; + optional string namespace = 3; +} + +message GetTableNamesResponse { + repeated TableName table_names = 1; +} + +message GetTableStateRequest { + required TableName table_name = 1; +} + +message GetTableStateResponse { + required TableState table_state = 1; +} + +message GetClusterStatusRequest { +} + +message GetClusterStatusResponse { + required ClusterStatus cluster_status = 1; +} + +message IsMasterRunningRequest { +} + +message IsMasterRunningResponse { + required bool is_master_running = 1; +} + +message ExecProcedureRequest { + required ProcedureDescription procedure = 1; +} + +message ExecProcedureResponse { + optional int64 expected_timeout = 1; + optional bytes return_data = 2; +} + +message IsProcedureDoneRequest { + optional ProcedureDescription procedure = 1; +} + +message IsProcedureDoneResponse { + optional bool done = 1 [default = false]; + optional ProcedureDescription snapshot = 2; +} + +message GetProcedureResultRequest { + required uint64 proc_id = 1; +} + +message GetProcedureResultResponse { + enum State { + NOT_FOUND = 0; + RUNNING = 1; + FINISHED = 2; + } + + required State state = 1; + optional uint64 start_time = 2; + optional uint64 last_update = 3; + optional bytes result = 4; + optional ForeignExceptionMessage exception = 5; +} + +message AbortProcedureRequest { + required uint64 proc_id = 1; + optional bool mayInterruptIfRunning = 2 [default = true]; +} + +message AbortProcedureResponse { + required bool is_procedure_aborted = 1; +} + +message ListProceduresRequest { +} + +message ListProceduresResponse { + repeated Procedure procedure = 1; +} + +message SetQuotaRequest { + optional string user_name = 1; + optional string user_group = 2; + optional string namespace = 3; + optional TableName table_name = 4; + + optional bool remove_all = 5; + optional bool bypass_globals = 6; + optional ThrottleRequest throttle = 7; +} + +message SetQuotaResponse { +} + +message MajorCompactionTimestampRequest { + required TableName table_name = 1; +} + +message MajorCompactionTimestampForRegionRequest { + required RegionSpecifier region = 1; +} + +message MajorCompactionTimestampResponse { + required int64 compaction_timestamp = 1; +} + +message SecurityCapabilitiesRequest { +} + +message SecurityCapabilitiesResponse { + enum Capability { + SIMPLE_AUTHENTICATION = 0; + SECURE_AUTHENTICATION = 1; + AUTHORIZATION = 2; + CELL_AUTHORIZATION = 3; + CELL_VISIBILITY = 4; + } + + repeated Capability capabilities = 1; +} + +service MasterService { + /** Used by the client to get the number of regions that have received the updated schema */ + rpc GetSchemaAlterStatus(GetSchemaAlterStatusRequest) + returns(GetSchemaAlterStatusResponse); + + /** Get list of TableDescriptors for requested tables. */ + rpc GetTableDescriptors(GetTableDescriptorsRequest) + returns(GetTableDescriptorsResponse); + + /** Get the list of table names. */ + rpc GetTableNames(GetTableNamesRequest) + returns(GetTableNamesResponse); + + /** Return cluster status. */ + rpc GetClusterStatus(GetClusterStatusRequest) + returns(GetClusterStatusResponse); + + /** return true if master is available */ + rpc IsMasterRunning(IsMasterRunningRequest) returns(IsMasterRunningResponse); + + /** Adds a column to the specified table. */ + rpc AddColumn(AddColumnRequest) + returns(AddColumnResponse); + + /** Deletes a column from the specified table. Table must be disabled. */ + rpc DeleteColumn(DeleteColumnRequest) + returns(DeleteColumnResponse); + + /** Modifies an existing column on the specified table. */ + rpc ModifyColumn(ModifyColumnRequest) + returns(ModifyColumnResponse); + + /** Move the region region to the destination server. */ + rpc MoveRegion(MoveRegionRequest) + returns(MoveRegionResponse); + + /** Master dispatch merging the regions */ + rpc DispatchMergingRegions(DispatchMergingRegionsRequest) + returns(DispatchMergingRegionsResponse); + + /** Assign a region to a server chosen at random. */ + rpc AssignRegion(AssignRegionRequest) + returns(AssignRegionResponse); + + /** + * Unassign a region from current hosting regionserver. Region will then be + * assigned to a regionserver chosen at random. Region could be reassigned + * back to the same server. Use MoveRegion if you want + * to control the region movement. + */ + rpc UnassignRegion(UnassignRegionRequest) + returns(UnassignRegionResponse); + + /** + * Offline a region from the assignment manager's in-memory state. The + * region should be in a closed state and there will be no attempt to + * automatically reassign the region as in unassign. This is a special + * method, and should only be used by experts or hbck. + */ + rpc OfflineRegion(OfflineRegionRequest) + returns(OfflineRegionResponse); + + /** Deletes a table */ + rpc DeleteTable(DeleteTableRequest) + returns(DeleteTableResponse); + + /** Truncate a table */ + rpc truncateTable(TruncateTableRequest) + returns(TruncateTableResponse); + + /** Puts the table on-line (only needed if table has been previously taken offline) */ + rpc EnableTable(EnableTableRequest) + returns(EnableTableResponse); + + /** Take table offline */ + rpc DisableTable(DisableTableRequest) + returns(DisableTableResponse); + + /** Modify a table's metadata */ + rpc ModifyTable(ModifyTableRequest) + returns(ModifyTableResponse); + + /** Creates a new table asynchronously */ + rpc CreateTable(CreateTableRequest) + returns(CreateTableResponse); + + /** Shutdown an HBase cluster. */ + rpc Shutdown(ShutdownRequest) + returns(ShutdownResponse); + + /** Stop HBase Master only. Does not shutdown the cluster. */ + rpc StopMaster(StopMasterRequest) + returns(StopMasterResponse); + + /** + * Run the balancer. Will run the balancer and if regions to move, it will + * go ahead and do the reassignments. Can NOT run for various reasons. + * Check logs. + */ + rpc Balance(BalanceRequest) + returns(BalanceResponse); + + /** + * Turn the load balancer on or off. + * If synchronous is true, it waits until current balance() call, if outstanding, to return. + */ + rpc SetBalancerRunning(SetBalancerRunningRequest) + returns(SetBalancerRunningResponse); + + /** + * Query whether the Region Balancer is running. + */ + rpc IsBalancerEnabled(IsBalancerEnabledRequest) + returns(IsBalancerEnabledResponse); + + /** + * Run region normalizer. Can NOT run for various reasons. Check logs. + */ + rpc Normalize(NormalizeRequest) + returns(NormalizeResponse); + + /** + * Turn region normalizer on or off. + */ + rpc SetNormalizerRunning(SetNormalizerRunningRequest) + returns(SetNormalizerRunningResponse); + + /** + * Query whether region normalizer is enabled. + */ + rpc IsNormalizerEnabled(IsNormalizerEnabledRequest) + returns(IsNormalizerEnabledResponse); + + /** Get a run of the catalog janitor */ + rpc RunCatalogScan(RunCatalogScanRequest) + returns(RunCatalogScanResponse); + + /** + * Enable the catalog janitor on or off. + */ + rpc EnableCatalogJanitor(EnableCatalogJanitorRequest) + returns(EnableCatalogJanitorResponse); + + /** + * Query whether the catalog janitor is enabled. + */ + rpc IsCatalogJanitorEnabled(IsCatalogJanitorEnabledRequest) + returns(IsCatalogJanitorEnabledResponse); + + /** + * Call a master coprocessor endpoint + */ + rpc ExecMasterService(CoprocessorServiceRequest) + returns(CoprocessorServiceResponse); + + /** + * Create a snapshot for the given table. + */ + rpc Snapshot(SnapshotRequest) returns(SnapshotResponse); + + /** + * Get completed snapshots. + * Returns a list of snapshot descriptors for completed snapshots + */ + rpc GetCompletedSnapshots(GetCompletedSnapshotsRequest) returns(GetCompletedSnapshotsResponse); + + /** + * Delete an existing snapshot. This method can also be used to clean up an aborted snapshot. + */ + rpc DeleteSnapshot(DeleteSnapshotRequest) returns(DeleteSnapshotResponse); + + /** + * Determine if the snapshot is done yet. + */ + rpc IsSnapshotDone(IsSnapshotDoneRequest) returns(IsSnapshotDoneResponse); + + /** + * Restore a snapshot + */ + rpc RestoreSnapshot(RestoreSnapshotRequest) returns(RestoreSnapshotResponse); + + /** + * Determine if the snapshot restore is done yet. + */ + rpc IsRestoreSnapshotDone(IsRestoreSnapshotDoneRequest) returns(IsRestoreSnapshotDoneResponse); + + /** + * Execute a distributed procedure. + */ + rpc ExecProcedure(ExecProcedureRequest) returns(ExecProcedureResponse); + + /** + * Execute a distributed procedure with return data. + */ + rpc ExecProcedureWithRet(ExecProcedureRequest) returns(ExecProcedureResponse); + + /** + * Determine if the procedure is done yet. + */ + rpc IsProcedureDone(IsProcedureDoneRequest) returns(IsProcedureDoneResponse); + + /** return true if master is available */ + /** rpc IsMasterRunning(IsMasterRunningRequest) returns(IsMasterRunningResponse); */ + + /** Modify a namespace's metadata */ + rpc ModifyNamespace(ModifyNamespaceRequest) + returns(ModifyNamespaceResponse); + + /** Creates a new namespace synchronously */ + rpc CreateNamespace(CreateNamespaceRequest) + returns(CreateNamespaceResponse); + + /** Deletes namespace synchronously */ + rpc DeleteNamespace(DeleteNamespaceRequest) + returns(DeleteNamespaceResponse); + + /** Get a namespace descriptor by name */ + rpc GetNamespaceDescriptor(GetNamespaceDescriptorRequest) + returns(GetNamespaceDescriptorResponse); + + /** returns a list of namespaces */ + rpc ListNamespaceDescriptors(ListNamespaceDescriptorsRequest) + returns(ListNamespaceDescriptorsResponse); + + /** returns a list of tables for a given namespace*/ + rpc ListTableDescriptorsByNamespace(ListTableDescriptorsByNamespaceRequest) + returns(ListTableDescriptorsByNamespaceResponse); + + /** returns a list of tables for a given namespace*/ + rpc ListTableNamesByNamespace(ListTableNamesByNamespaceRequest) + returns(ListTableNamesByNamespaceResponse); + + /** returns table state */ + rpc GetTableState(GetTableStateRequest) + returns(GetTableStateResponse); + + /** Apply the new quota settings */ + rpc SetQuota(SetQuotaRequest) returns(SetQuotaResponse); + + /** Returns the timestamp of the last major compaction */ + rpc getLastMajorCompactionTimestamp(MajorCompactionTimestampRequest) + returns(MajorCompactionTimestampResponse); + + /** Returns the timestamp of the last major compaction */ + rpc getLastMajorCompactionTimestampForRegion(MajorCompactionTimestampForRegionRequest) + returns(MajorCompactionTimestampResponse); + + rpc getProcedureResult(GetProcedureResultRequest) + returns(GetProcedureResultResponse); + + /** Returns the security capabilities in effect on the cluster */ + rpc getSecurityCapabilities(SecurityCapabilitiesRequest) + returns(SecurityCapabilitiesResponse); + + /** Abort a procedure */ + rpc AbortProcedure(AbortProcedureRequest) + returns(AbortProcedureResponse); + + /** returns a list of procedures */ + rpc ListProcedures(ListProceduresRequest) + returns(ListProceduresResponse); +} diff --git a/hbase-native-client/if/MasterProcedure.proto b/hbase-native-client/if/MasterProcedure.proto new file mode 100644 index 0000000..2d2aff4 --- /dev/null +++ b/hbase-native-client/if/MasterProcedure.proto @@ -0,0 +1,245 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package hbase.pb; + +option java_package = "org.apache.hadoop.hbase.protobuf.generated"; +option java_outer_classname = "MasterProcedureProtos"; +option java_generic_services = true; +option java_generate_equals_and_hash = true; +option optimize_for = SPEED; + +import "HBase.proto"; +import "RPC.proto"; + +// ============================================================================ +// WARNING - Compatibility rules +// ============================================================================ +// This .proto contains the data serialized by the master procedures. +// Each procedure has some state stored to know, which step were executed +// and what were the parameters or data created by the previous steps. +// new code should be able to handle the old format or at least fail cleanly +// triggering a rollback/cleanup. +// +// Procedures that are inheriting from a StateMachineProcedure have an enum: +// - Do not change the number of the 'State' enums. +// doing so, will cause executing the wrong 'step' on the pending +// procedures when they will be replayed. +// - Do not remove items from the enum, new code must be able to handle +// all the previous 'steps'. There may be pending procedure ready to be +// recovered replayed. alternative you can make sure that not-known state +// will result in a failure that will rollback the already executed steps. +// ============================================================================ + +enum CreateTableState { + CREATE_TABLE_PRE_OPERATION = 1; + CREATE_TABLE_WRITE_FS_LAYOUT = 2; + CREATE_TABLE_ADD_TO_META = 3; + CREATE_TABLE_ASSIGN_REGIONS = 4; + CREATE_TABLE_UPDATE_DESC_CACHE = 5; + CREATE_TABLE_POST_OPERATION = 6; +} + +message CreateTableStateData { + required UserInformation user_info = 1; + required TableSchema table_schema = 2; + repeated RegionInfo region_info = 3; +} + +enum ModifyTableState { + MODIFY_TABLE_PREPARE = 1; + MODIFY_TABLE_PRE_OPERATION = 2; + MODIFY_TABLE_UPDATE_TABLE_DESCRIPTOR = 3; + MODIFY_TABLE_REMOVE_REPLICA_COLUMN = 4; + MODIFY_TABLE_DELETE_FS_LAYOUT = 5; + MODIFY_TABLE_POST_OPERATION = 6; + MODIFY_TABLE_REOPEN_ALL_REGIONS = 7; +} + +message ModifyTableStateData { + required UserInformation user_info = 1; + optional TableSchema unmodified_table_schema = 2; + required TableSchema modified_table_schema = 3; + required bool delete_column_family_in_modify = 4; +} + +enum TruncateTableState { + TRUNCATE_TABLE_PRE_OPERATION = 1; + TRUNCATE_TABLE_REMOVE_FROM_META = 2; + TRUNCATE_TABLE_CLEAR_FS_LAYOUT = 3; + TRUNCATE_TABLE_CREATE_FS_LAYOUT = 4; + TRUNCATE_TABLE_ADD_TO_META = 5; + TRUNCATE_TABLE_ASSIGN_REGIONS = 6; + TRUNCATE_TABLE_POST_OPERATION = 7; +} + +message TruncateTableStateData { + required UserInformation user_info = 1; + required bool preserve_splits = 2; + optional TableName table_name = 3; + optional TableSchema table_schema = 4; + repeated RegionInfo region_info = 5; +} + +enum DeleteTableState { + DELETE_TABLE_PRE_OPERATION = 1; + DELETE_TABLE_REMOVE_FROM_META = 2; + DELETE_TABLE_CLEAR_FS_LAYOUT = 3; + DELETE_TABLE_UPDATE_DESC_CACHE = 4; + DELETE_TABLE_UNASSIGN_REGIONS = 5; + DELETE_TABLE_POST_OPERATION = 6; +} + +message DeleteTableStateData { + required UserInformation user_info = 1; + required TableName table_name = 2; + repeated RegionInfo region_info = 3; +} + +enum CreateNamespaceState { + CREATE_NAMESPACE_PREPARE = 1; + CREATE_NAMESPACE_CREATE_DIRECTORY = 2; + CREATE_NAMESPACE_INSERT_INTO_NS_TABLE = 3; + CREATE_NAMESPACE_UPDATE_ZK = 4; + CREATE_NAMESPACE_SET_NAMESPACE_QUOTA = 5; +} + +message CreateNamespaceStateData { + required NamespaceDescriptor namespace_descriptor = 1; +} + +enum ModifyNamespaceState { + MODIFY_NAMESPACE_PREPARE = 1; + MODIFY_NAMESPACE_UPDATE_NS_TABLE = 2; + MODIFY_NAMESPACE_UPDATE_ZK = 3; +} + +message ModifyNamespaceStateData { + required NamespaceDescriptor namespace_descriptor = 1; + optional NamespaceDescriptor unmodified_namespace_descriptor = 2; +} + +enum DeleteNamespaceState { + DELETE_NAMESPACE_PREPARE = 1; + DELETE_NAMESPACE_DELETE_FROM_NS_TABLE = 2; + DELETE_NAMESPACE_REMOVE_FROM_ZK = 3; + DELETE_NAMESPACE_DELETE_DIRECTORIES = 4; + DELETE_NAMESPACE_REMOVE_NAMESPACE_QUOTA = 5; +} + +message DeleteNamespaceStateData { + required string namespace_name = 1; + optional NamespaceDescriptor namespace_descriptor = 2; +} + +enum AddColumnFamilyState { + ADD_COLUMN_FAMILY_PREPARE = 1; + ADD_COLUMN_FAMILY_PRE_OPERATION = 2; + ADD_COLUMN_FAMILY_UPDATE_TABLE_DESCRIPTOR = 3; + ADD_COLUMN_FAMILY_POST_OPERATION = 4; + ADD_COLUMN_FAMILY_REOPEN_ALL_REGIONS = 5; +} + +message AddColumnFamilyStateData { + required UserInformation user_info = 1; + required TableName table_name = 2; + required ColumnFamilySchema columnfamily_schema = 3; + optional TableSchema unmodified_table_schema = 4; +} + +enum ModifyColumnFamilyState { + MODIFY_COLUMN_FAMILY_PREPARE = 1; + MODIFY_COLUMN_FAMILY_PRE_OPERATION = 2; + MODIFY_COLUMN_FAMILY_UPDATE_TABLE_DESCRIPTOR = 3; + MODIFY_COLUMN_FAMILY_POST_OPERATION = 4; + MODIFY_COLUMN_FAMILY_REOPEN_ALL_REGIONS = 5; +} + +message ModifyColumnFamilyStateData { + required UserInformation user_info = 1; + required TableName table_name = 2; + required ColumnFamilySchema columnfamily_schema = 3; + optional TableSchema unmodified_table_schema = 4; +} + +enum DeleteColumnFamilyState { + DELETE_COLUMN_FAMILY_PREPARE = 1; + DELETE_COLUMN_FAMILY_PRE_OPERATION = 2; + DELETE_COLUMN_FAMILY_UPDATE_TABLE_DESCRIPTOR = 3; + DELETE_COLUMN_FAMILY_DELETE_FS_LAYOUT = 4; + DELETE_COLUMN_FAMILY_POST_OPERATION = 5; + DELETE_COLUMN_FAMILY_REOPEN_ALL_REGIONS = 6; +} + +message DeleteColumnFamilyStateData { + required UserInformation user_info = 1; + required TableName table_name = 2; + required bytes columnfamily_name = 3; + optional TableSchema unmodified_table_schema = 4; +} + +enum EnableTableState { + ENABLE_TABLE_PREPARE = 1; + ENABLE_TABLE_PRE_OPERATION = 2; + ENABLE_TABLE_SET_ENABLING_TABLE_STATE = 3; + ENABLE_TABLE_MARK_REGIONS_ONLINE = 4; + ENABLE_TABLE_SET_ENABLED_TABLE_STATE = 5; + ENABLE_TABLE_POST_OPERATION = 6; +} + +message EnableTableStateData { + required UserInformation user_info = 1; + required TableName table_name = 2; + required bool skip_table_state_check = 3; +} + +enum DisableTableState { + DISABLE_TABLE_PREPARE = 1; + DISABLE_TABLE_PRE_OPERATION = 2; + DISABLE_TABLE_SET_DISABLING_TABLE_STATE = 3; + DISABLE_TABLE_MARK_REGIONS_OFFLINE = 4; + DISABLE_TABLE_SET_DISABLED_TABLE_STATE = 5; + DISABLE_TABLE_POST_OPERATION = 6; +} + +message DisableTableStateData { + required UserInformation user_info = 1; + required TableName table_name = 2; + required bool skip_table_state_check = 3; +} + +message ServerCrashStateData { + required ServerName server_name = 1; + optional bool distributed_log_replay = 2; + repeated RegionInfo regions_on_crashed_server = 3; + repeated RegionInfo regions_assigned = 4; + optional bool carrying_meta = 5; + optional bool should_split_wal = 6 [default = true]; +} + +enum ServerCrashState { + SERVER_CRASH_START = 1; + SERVER_CRASH_PROCESS_META = 2; + SERVER_CRASH_GET_REGIONS = 3; + SERVER_CRASH_NO_SPLIT_LOGS = 4; + SERVER_CRASH_SPLIT_LOGS = 5; + SERVER_CRASH_PREPARE_LOG_REPLAY = 6; + // Removed SERVER_CRASH_CALC_REGIONS_TO_ASSIGN = 7; + SERVER_CRASH_ASSIGN = 8; + SERVER_CRASH_WAIT_ON_ASSIGN = 9; + SERVER_CRASH_FINISH = 100; +} diff --git a/hbase-native-client/if/MultiRowMutation.proto b/hbase-native-client/if/MultiRowMutation.proto new file mode 100644 index 0000000..747afac --- /dev/null +++ b/hbase-native-client/if/MultiRowMutation.proto @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package hbase.pb; + +import "Client.proto"; +option java_package = "org.apache.hadoop.hbase.protobuf.generated"; +option java_outer_classname = "MultiRowMutationProtos"; +option java_generate_equals_and_hash = true; +option java_generic_services = true; +option optimize_for = SPEED; + +message MultiRowMutationProcessorRequest{ +} + +message MultiRowMutationProcessorResponse{ +} + +message MutateRowsRequest { + repeated MutationProto mutation_request = 1; + optional uint64 nonce_group = 2; + optional uint64 nonce = 3; +} + +message MutateRowsResponse { +} + +service MultiRowMutationService { + rpc MutateRows(MutateRowsRequest) + returns(MutateRowsResponse); +} \ No newline at end of file diff --git a/hbase-native-client/if/Procedure.proto b/hbase-native-client/if/Procedure.proto new file mode 100644 index 0000000..55e44a4 --- /dev/null +++ b/hbase-native-client/if/Procedure.proto @@ -0,0 +1,119 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package hbase.pb; + +option java_package = "org.apache.hadoop.hbase.protobuf.generated"; +option java_outer_classname = "ProcedureProtos"; +option java_generic_services = true; +option java_generate_equals_and_hash = true; +option optimize_for = SPEED; + +import "ErrorHandling.proto"; + +enum ProcedureState { + INITIALIZING = 1; // Procedure in construction, not yet added to the executor + RUNNABLE = 2; // Procedure added to the executor, and ready to be executed + WAITING = 3; // The procedure is waiting on children to be completed + WAITING_TIMEOUT = 4; // The procedure is waiting a timout or an external event + ROLLEDBACK = 5; // The procedure failed and was rolledback + FINISHED = 6; // The procedure execution is completed. may need a rollback if failed. +} + +/** + * Procedure metadata, serialized by the ProcedureStore to be able to recover the old state. + */ +message Procedure { + // internal "static" state + required string class_name = 1; // full classname to be able to instantiate the procedure + optional uint64 parent_id = 2; // parent if not a root-procedure otherwise not set + required uint64 proc_id = 3; + required uint64 start_time = 4; + optional string owner = 5; + + // internal "runtime" state + required ProcedureState state = 6; + repeated uint32 stack_id = 7; // stack indices in case the procedure was running + required uint64 last_update = 8; + optional uint32 timeout = 9; + + // user state/results + optional ForeignExceptionMessage exception = 10; + optional bytes result = 11; // opaque (user) result structure + optional bytes state_data = 12; // opaque (user) procedure internal-state + + // Nonce to prevent same procedure submit by multiple times + optional uint64 nonce_group = 13 [default = 0]; + optional uint64 nonce = 14 [default = 0]; +} + +/** + * SequentialProcedure data + */ +message SequentialProcedureData { + required bool executed = 1; +} + +/** + * StateMachineProcedure data + */ +message StateMachineProcedureData { + repeated uint32 state = 1; +} + +/** + * Procedure WAL header + */ +message ProcedureWALHeader { + required uint32 version = 1; + required uint32 type = 2; + required uint64 log_id = 3; + required uint64 min_proc_id = 4; +} + +/** + * Procedure WAL trailer + */ +message ProcedureWALTrailer { + required uint32 version = 1; + required uint64 tracker_pos = 2; +} + +message ProcedureStoreTracker { + message TrackerNode { + required uint64 start_id = 1; + repeated uint64 updated = 2; + repeated uint64 deleted = 3; + } + + repeated TrackerNode node = 1; +} + +message ProcedureWALEntry { + enum Type { + PROCEDURE_WAL_EOF = 1; + PROCEDURE_WAL_INIT = 2; + PROCEDURE_WAL_INSERT = 3; + PROCEDURE_WAL_UPDATE = 4; + PROCEDURE_WAL_DELETE = 5; + PROCEDURE_WAL_COMPACT = 6; + } + + required Type type = 1; + repeated Procedure procedure = 2; + optional uint64 proc_id = 3; +} diff --git a/hbase-native-client/if/Quota.proto b/hbase-native-client/if/Quota.proto new file mode 100644 index 0000000..a8303b1 --- /dev/null +++ b/hbase-native-client/if/Quota.proto @@ -0,0 +1,76 @@ + /** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package hbase.pb; + +option java_package = "org.apache.hadoop.hbase.protobuf.generated"; +option java_outer_classname = "QuotaProtos"; +option java_generic_services = true; +option java_generate_equals_and_hash = true; +option optimize_for = SPEED; + +import "HBase.proto"; + +enum QuotaScope { + CLUSTER = 1; + MACHINE = 2; +} + +message TimedQuota { + required TimeUnit time_unit = 1; + optional uint64 soft_limit = 2; + optional float share = 3; + optional QuotaScope scope = 4 [default = MACHINE]; +} + +enum ThrottleType { + REQUEST_NUMBER = 1; + REQUEST_SIZE = 2; + WRITE_NUMBER = 3; + WRITE_SIZE = 4; + READ_NUMBER = 5; + READ_SIZE = 6; +} + +message Throttle { + optional TimedQuota req_num = 1; + optional TimedQuota req_size = 2; + + optional TimedQuota write_num = 3; + optional TimedQuota write_size = 4; + + optional TimedQuota read_num = 5; + optional TimedQuota read_size = 6; +} + +message ThrottleRequest { + optional ThrottleType type = 1; + optional TimedQuota timed_quota = 2; +} + +enum QuotaType { + THROTTLE = 1; +} + +message Quotas { + optional bool bypass_globals = 1 [default = false]; + optional Throttle throttle = 2; +} + +message QuotaUsage { +} diff --git a/hbase-native-client/if/RPC.proto b/hbase-native-client/if/RPC.proto new file mode 100644 index 0000000..59bb03d --- /dev/null +++ b/hbase-native-client/if/RPC.proto @@ -0,0 +1,136 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package hbase.pb; + +import "Tracing.proto"; +import "HBase.proto"; + +option java_package = "org.apache.hadoop.hbase.protobuf.generated"; +option java_outer_classname = "RPCProtos"; +option java_generate_equals_and_hash = true; +option optimize_for = SPEED; + +// See https://issues.apache.org/jira/browse/HBASE-7898 for high-level +// description of RPC specification. +// +// On connection setup, the client sends six bytes of preamble -- a four +// byte magic, a byte of version, and a byte of authentication type. +// +// We then send a "ConnectionHeader" protobuf of user information and the +// 'protocol' or 'service' that is to be run over this connection as well as +// info such as codecs and compression to use when we send cell blocks(see below). +// This connection header protobuf is prefaced by an int that holds the length +// of this connection header (this is NOT a varint). The pb connection header +// is sent with Message#writeTo. The server throws an exception if it doesn't +// like what it was sent noting what it is objecting too. Otherwise, the server +// says nothing and is open for business. +// +// Hereafter the client makes requests and the server returns responses. +// +// Requests look like this: +// +// +// +// +// +// +// ...where the Request Parameter Message is whatever the method name stipulated +// in the RequestHeader expects; e.g. if the method is a scan, then the pb +// Request Message is a GetRequest, or a ScanRequest. A block of Cells +// optionally follows. The presence of a Request param Message and/or a +// block of Cells will be noted in the RequestHeader. +// +// Response is the mirror of the request: +// +// +// +// +// +// +// ...where the Response Message is the response type that goes with the +// method specified when making the request and the follow on Cell blocks may +// or may not be there -- read the response header to find out if one following. +// If an exception, it will be included inside the Response Header. +// +// Any time we write a pb, we do it with Message#writeDelimitedTo EXCEPT when +// the connection header is sent; this is prefaced by an int with its length +// and the pb connection header is then written with Message#writeTo. +// + +// User Information proto. Included in ConnectionHeader on connection setup +message UserInformation { + required string effective_user = 1; + optional string real_user = 2; +} + +// This is sent on connection setup after the connection preamble is sent. +message ConnectionHeader { + optional UserInformation user_info = 1; + optional string service_name = 2; + // Cell block codec we will use sending over optional cell blocks. Server throws exception + // if cannot deal. Null means no codec'ing going on so we are pb all the time (SLOW!!!) + optional string cell_block_codec_class = 3; + // Compressor we will use if cell block is compressed. Server will throw exception if not supported. + // Class must implement hadoop's CompressionCodec Interface. Can't compress if no codec. + optional string cell_block_compressor_class = 4; + optional VersionInfo version_info = 5; +} + +// Optional Cell block Message. Included in client RequestHeader +message CellBlockMeta { + // Length of the following cell block. Could calculate it but convenient having it too hand. + optional uint32 length = 1; +} + +// At the RPC layer, this message is used to carry +// the server side exception to the RPC client. +message ExceptionResponse { + // Class name of the exception thrown from the server + optional string exception_class_name = 1; + // Exception stack trace from the server side + optional string stack_trace = 2; + // Optional hostname. Filled in for some exceptions such as region moved + // where exception gives clue on where the region may have moved. + optional string hostname = 3; + optional int32 port = 4; + // Set if we are NOT to retry on receipt of this exception + optional bool do_not_retry = 5; +} + +// Header sent making a request. +message RequestHeader { + // Monotonically increasing call_id to keep track of RPC requests and their response + optional uint32 call_id = 1; + optional RPCTInfo trace_info = 2; + optional string method_name = 3; + // If true, then a pb Message param follows. + optional bool request_param = 4; + // If present, then an encoded data block follows. + optional CellBlockMeta cell_block_meta = 5; + // 0 is NORMAL priority. 200 is HIGH. If no priority, treat it as NORMAL. + // See HConstants. + optional uint32 priority = 6; +} + +message ResponseHeader { + optional uint32 call_id = 1; + // If present, then request threw an exception and no response message (else we presume one) + optional ExceptionResponse exception = 2; + // If present, then an encoded data block follows. + optional CellBlockMeta cell_block_meta = 3; +} diff --git a/hbase-native-client/if/RegionNormalizer.proto b/hbase-native-client/if/RegionNormalizer.proto new file mode 100644 index 0000000..e5305d6 --- /dev/null +++ b/hbase-native-client/if/RegionNormalizer.proto @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file contains protocol buffers to represent the state of the load balancer. + +option java_package = "org.apache.hadoop.hbase.protobuf.generated"; +option java_outer_classname = "RegionNormalizerProtos"; +option java_generate_equals_and_hash = true; +option optimize_for = SPEED; + +message RegionNormalizerState { + optional bool normalizer_on = 1; +} diff --git a/hbase-native-client/if/RegionServerStatus.proto b/hbase-native-client/if/RegionServerStatus.proto new file mode 100644 index 0000000..fda9de2 --- /dev/null +++ b/hbase-native-client/if/RegionServerStatus.proto @@ -0,0 +1,158 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file contains protocol buffers that are used for RegionServerStatusProtocol. +package hbase.pb; + +option java_package = "org.apache.hadoop.hbase.protobuf.generated"; +option java_outer_classname = "RegionServerStatusProtos"; +option java_generic_services = true; +option java_generate_equals_and_hash = true; +option optimize_for = SPEED; + +import "HBase.proto"; +import "ClusterStatus.proto"; + +message RegionServerStartupRequest { + /** Port number this regionserver is up on */ + required uint32 port = 1; + + /** This servers' startcode */ + required uint64 server_start_code = 2; + + /** Current time of the region server in ms */ + required uint64 server_current_time = 3; + + /** hostname for region server, optional */ + optional string use_this_hostname_instead = 4; +} + +message RegionServerStartupResponse { + /** + * Configuration for the regionserver to use: e.g. filesystem, + * hbase rootdir, the hostname to use creating the RegionServer ServerName, + * etc + */ + repeated NameStringPair map_entries = 1; +} + +message RegionServerReportRequest { + required ServerName server = 1; + + /** load the server is under */ + optional ServerLoad load = 2; +} + +message RegionServerReportResponse { +} + +message ReportRSFatalErrorRequest { + /** name of the server experiencing the error */ + required ServerName server = 1; + + /** informative text to expose in the master logs and UI */ + required string error_message = 2; +} + +message ReportRSFatalErrorResponse { +} + +message GetLastFlushedSequenceIdRequest { + /** region name */ + required bytes region_name = 1; +} + +message GetLastFlushedSequenceIdResponse { + /** the last WAL sequence id flushed from MemStore to HFile for the region */ + required uint64 last_flushed_sequence_id = 1; + + /** the last WAL sequence id flushed from MemStore to HFile for stores of the region */ + repeated StoreSequenceId store_last_flushed_sequence_id = 2; +} + +message RegionStateTransition { + required TransitionCode transition_code = 1; + + /** Mutliple regions are involved during merging/splitting */ + repeated RegionInfo region_info = 2; + + /** For newly opened region, the open seq num is needed */ + optional uint64 open_seq_num = 3; + + enum TransitionCode { + OPENED = 0; + FAILED_OPEN = 1; + /** No failed_close, in which case region server will abort */ + CLOSED = 2; + + /** Ask master for ok to split/merge region(s) */ + READY_TO_SPLIT = 3; + READY_TO_MERGE = 4; + + SPLIT_PONR = 5; + MERGE_PONR = 6; + + SPLIT = 7; + MERGED = 8; + SPLIT_REVERTED = 9; + MERGE_REVERTED = 10; + } +} + +message ReportRegionStateTransitionRequest { + /** This region server's server name */ + required ServerName server = 1; + + repeated RegionStateTransition transition = 2; +} + +message ReportRegionStateTransitionResponse { + /** Error message if failed to update the region state */ + optional string error_message = 1; +} + +service RegionServerStatusService { + /** Called when a region server first starts. */ + rpc RegionServerStartup(RegionServerStartupRequest) + returns(RegionServerStartupResponse); + + /** Called to report the load the RegionServer is under. */ + rpc RegionServerReport(RegionServerReportRequest) + returns(RegionServerReportResponse); + + /** + * Called by a region server to report a fatal error that is causing it to + * abort. + */ + rpc ReportRSFatalError(ReportRSFatalErrorRequest) + returns(ReportRSFatalErrorResponse); + + /** Called to get the sequence id of the last MemStore entry flushed to an + * HFile for a specified region. Used by the region server to speed up + * log splitting. */ + rpc GetLastFlushedSequenceId(GetLastFlushedSequenceIdRequest) + returns(GetLastFlushedSequenceIdResponse); + + /** + * Called by a region server to report the progress of a region + * transition. If the request fails, the transition should + * be aborted. + */ + rpc ReportRegionStateTransition(ReportRegionStateTransitionRequest) + returns(ReportRegionStateTransitionResponse); +} diff --git a/hbase-native-client/if/RowProcessor.proto b/hbase-native-client/if/RowProcessor.proto new file mode 100644 index 0000000..cf2f30f --- /dev/null +++ b/hbase-native-client/if/RowProcessor.proto @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Defines a protocol to perform multi row transactions. + * See BaseRowProcessorEndpoint for the implementation. + * See HRegion#processRowsWithLocks() for details. + */ +package hbase.pb; + +option java_package = "org.apache.hadoop.hbase.protobuf.generated"; +option java_outer_classname = "RowProcessorProtos"; +option java_generic_services = true; +option java_generate_equals_and_hash = true; +option optimize_for = SPEED; + +message ProcessRequest { + required string row_processor_class_name = 1; + optional string row_processor_initializer_message_name = 2; + optional bytes row_processor_initializer_message = 3; + optional uint64 nonce_group = 4; + optional uint64 nonce = 5; +} + +message ProcessResponse { + required bytes row_processor_result = 1; +} + +service RowProcessorService { + rpc Process(ProcessRequest) returns (ProcessResponse); +} diff --git a/hbase-native-client/if/SecureBulkLoad.proto b/hbase-native-client/if/SecureBulkLoad.proto new file mode 100644 index 0000000..814735b --- /dev/null +++ b/hbase-native-client/if/SecureBulkLoad.proto @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package hbase.pb; + +option java_package = "org.apache.hadoop.hbase.protobuf.generated"; +option java_outer_classname = "SecureBulkLoadProtos"; +option java_generic_services = true; +option java_generate_equals_and_hash = true; +option optimize_for = SPEED; + +import 'HBase.proto'; +import 'Client.proto'; + +message SecureBulkLoadHFilesRequest { + repeated BulkLoadHFileRequest.FamilyPath family_path = 1; + optional bool assign_seq_num = 2; + required DelegationToken fs_token = 3; + required string bulk_token = 4; +} + +message SecureBulkLoadHFilesResponse { + required bool loaded = 1; +} + +message DelegationToken { + optional bytes identifier = 1; + optional bytes password = 2; + optional string kind = 3; + optional string service = 4; +} + +message PrepareBulkLoadRequest { + required TableName table_name = 1; +} + +message PrepareBulkLoadResponse { + required string bulk_token = 1; +} + +message CleanupBulkLoadRequest { + required string bulk_token = 1; + +} + +message CleanupBulkLoadResponse { +} + +service SecureBulkLoadService { + rpc PrepareBulkLoad(PrepareBulkLoadRequest) + returns (PrepareBulkLoadResponse); + + rpc SecureBulkLoadHFiles(SecureBulkLoadHFilesRequest) + returns (SecureBulkLoadHFilesResponse); + + rpc CleanupBulkLoad(CleanupBulkLoadRequest) + returns (CleanupBulkLoadResponse); +} diff --git a/hbase-native-client/if/Snapshot.proto b/hbase-native-client/if/Snapshot.proto new file mode 100644 index 0000000..ae1a1e6 --- /dev/null +++ b/hbase-native-client/if/Snapshot.proto @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package hbase.pb; + +option java_package = "org.apache.hadoop.hbase.protobuf.generated"; +option java_outer_classname = "SnapshotProtos"; +option java_generic_services = true; +option java_generate_equals_and_hash = true; +option optimize_for = SPEED; + +import "FS.proto"; +import "HBase.proto"; + +message SnapshotFileInfo { + enum Type { + HFILE = 1; + WAL = 2; + } + + required Type type = 1; + + optional string hfile = 3; + + optional string wal_server = 4; + optional string wal_name = 5; +} + +message SnapshotRegionManifest { + optional int32 version = 1; + + required RegionInfo region_info = 2; + repeated FamilyFiles family_files = 3; + + message StoreFile { + required string name = 1; + optional Reference reference = 2; + + // TODO: Add checksums or other fields to verify the file + optional uint64 file_size = 3; + } + + message FamilyFiles { + required bytes family_name = 1; + repeated StoreFile store_files = 2; + } +} + +message SnapshotDataManifest { + required TableSchema table_schema = 1; + repeated SnapshotRegionManifest region_manifests = 2; +} diff --git a/hbase-native-client/if/Tracing.proto b/hbase-native-client/if/Tracing.proto new file mode 100644 index 0000000..5a64cfc --- /dev/null +++ b/hbase-native-client/if/Tracing.proto @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package hbase.pb; + +option java_package = "org.apache.hadoop.hbase.protobuf.generated"; +option java_outer_classname = "TracingProtos"; +option java_generate_equals_and_hash = true; +option optimize_for = SPEED; + +//Used to pass through the information necessary to continue +//a trace after an RPC is made. All we need is the traceid +//(so we know the overarching trace this message is a part of), and +//the id of the current span when this message was sent, so we know +//what span caused the new span we will create when this message is received. +message RPCTInfo { + optional int64 trace_id = 1; + optional int64 parent_id = 2; +} diff --git a/hbase-native-client/if/VisibilityLabels.proto b/hbase-native-client/if/VisibilityLabels.proto new file mode 100644 index 0000000..d2dc44d --- /dev/null +++ b/hbase-native-client/if/VisibilityLabels.proto @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package hbase.pb; + +option java_package = "org.apache.hadoop.hbase.protobuf.generated"; +option java_outer_classname = "VisibilityLabelsProtos"; +option java_generic_services = true; +option java_generate_equals_and_hash = true; +option optimize_for = SPEED; + +import "Client.proto"; + +message VisibilityLabelsRequest { + repeated VisibilityLabel visLabel = 1; +} + +message VisibilityLabel { + required bytes label = 1; + optional uint32 ordinal = 2; +} + +message VisibilityLabelsResponse { + repeated RegionActionResult result = 1; +} + +message SetAuthsRequest { + required bytes user = 1; + repeated bytes auth = 2; +} + +message UserAuthorizations { + required bytes user = 1; + repeated uint32 auth = 2; +} + +message MultiUserAuthorizations { + repeated UserAuthorizations userAuths = 1; +} + +message GetAuthsRequest { + required bytes user = 1; +} + +message GetAuthsResponse { + required bytes user = 1; + repeated bytes auth = 2; +} + +message ListLabelsRequest { + optional string regex = 1; +} + +message ListLabelsResponse { + repeated bytes label = 1; +} + +service VisibilityLabelsService { + rpc addLabels(VisibilityLabelsRequest) + returns (VisibilityLabelsResponse); + rpc setAuths(SetAuthsRequest) + returns (VisibilityLabelsResponse); + rpc clearAuths(SetAuthsRequest) + returns (VisibilityLabelsResponse); + rpc getAuths(GetAuthsRequest) + returns (GetAuthsResponse); + rpc listLabels(ListLabelsRequest) + returns (ListLabelsResponse); +} \ No newline at end of file diff --git a/hbase-native-client/if/WAL.proto b/hbase-native-client/if/WAL.proto new file mode 100644 index 0000000..cb9bd8f --- /dev/null +++ b/hbase-native-client/if/WAL.proto @@ -0,0 +1,172 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package hbase.pb; + +option java_package = "org.apache.hadoop.hbase.protobuf.generated"; +option java_outer_classname = "WALProtos"; +option java_generic_services = false; +option java_generate_equals_and_hash = true; +option optimize_for = SPEED; + +import "HBase.proto"; +import "Client.proto"; + +message WALHeader { + optional bool has_compression = 1; + optional bytes encryption_key = 2; + optional bool has_tag_compression = 3; + optional string writer_cls_name = 4; + optional string cell_codec_cls_name = 5; +} + +/* + * Protocol buffer version of WALKey; see WALKey comment, not really a key but WALEdit header + * for some KVs + */ +message WALKey { + required bytes encoded_region_name = 1; + required bytes table_name = 2; + required uint64 log_sequence_number = 3; + required uint64 write_time = 4; + /* + This parameter is deprecated in favor of clusters which + contains the list of clusters that have consumed the change. + It is retained so that the log created by earlier releases (0.94) + can be read by the newer releases. + */ + optional UUID cluster_id = 5 [deprecated=true]; + + repeated FamilyScope scopes = 6; + optional uint32 following_kv_count = 7; + + /* + This field contains the list of clusters that have + consumed the change + */ + repeated UUID cluster_ids = 8; + + optional uint64 nonceGroup = 9; + optional uint64 nonce = 10; + optional uint64 orig_sequence_number = 11; + +/* + optional CustomEntryType custom_entry_type = 9; + + enum CustomEntryType { + COMPACTION = 0; + } +*/ +} + +enum ScopeType { + REPLICATION_SCOPE_LOCAL = 0; + REPLICATION_SCOPE_GLOBAL = 1; +} + +message FamilyScope { + required bytes family = 1; + required ScopeType scope_type = 2; +} + +/** + * Custom WAL entries + */ + +/** + * Special WAL entry to hold all related to a compaction. + * Written to WAL before completing compaction. There is + * sufficient info in the below message to complete later + * the * compaction should we fail the WAL write. + */ +message CompactionDescriptor { + required bytes table_name = 1; // TODO: WALKey already stores these, might remove + required bytes encoded_region_name = 2; + required bytes family_name = 3; + repeated string compaction_input = 4; // relative to store dir + repeated string compaction_output = 5; + required string store_home_dir = 6; // relative to region dir + optional bytes region_name = 7; // full region name +} + +/** + * Special WAL entry to hold all related to a flush. + */ +message FlushDescriptor { + enum FlushAction { + START_FLUSH = 0; + COMMIT_FLUSH = 1; + ABORT_FLUSH = 2; + CANNOT_FLUSH = 3; // marker for indicating that a flush has been requested but cannot complete + } + + message StoreFlushDescriptor { + required bytes family_name = 1; + required string store_home_dir = 2; //relative to region dir + repeated string flush_output = 3; // relative to store dir (if this is a COMMIT_FLUSH) + } + + required FlushAction action = 1; + required bytes table_name = 2; + required bytes encoded_region_name = 3; + optional uint64 flush_sequence_number = 4; + repeated StoreFlushDescriptor store_flushes = 5; + optional bytes region_name = 6; // full region name +} + +message StoreDescriptor { + required bytes family_name = 1; + required string store_home_dir = 2; //relative to region dir + repeated string store_file = 3; // relative to store dir +} + +/** + * Special WAL entry used for writing bulk load events to WAL + */ +message BulkLoadDescriptor { + required TableName table_name = 1; + required bytes encoded_region_name = 2; + repeated StoreDescriptor stores = 3; + required int64 bulkload_seq_num = 4; +} + +/** + * Special WAL entry to hold all related to a region event (open/close). + */ +message RegionEventDescriptor { + enum EventType { + REGION_OPEN = 0; + REGION_CLOSE = 1; + } + + required EventType event_type = 1; + required bytes table_name = 2; + required bytes encoded_region_name = 3; + optional uint64 log_sequence_number = 4; + repeated StoreDescriptor stores = 5; + optional ServerName server = 6; // Server who opened the region + optional bytes region_name = 7; // full region name +} + +/** + * A trailer that is appended to the end of a properly closed WAL file. + * If missing, this is either a legacy or a corrupted WAL file. + * N.B. This trailer currently doesn't contain any information and we + * purposefully don't expose it in the WAL APIs. It's for future growth. + */ +message WALTrailer { +} diff --git a/hbase-native-client/if/ZooKeeper.proto b/hbase-native-client/if/ZooKeeper.proto new file mode 100644 index 0000000..54652af --- /dev/null +++ b/hbase-native-client/if/ZooKeeper.proto @@ -0,0 +1,155 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// ZNode data in hbase are serialized protobufs with a four byte +// 'magic' 'PBUF' prefix. +package hbase.pb; + +option java_package = "org.apache.hadoop.hbase.protobuf.generated"; +option java_outer_classname = "ZooKeeperProtos"; +option java_generic_services = true; +option java_generate_equals_and_hash = true; +option optimize_for = SPEED; + +import "HBase.proto"; +import "ClusterStatus.proto"; + +/** + * Content of the meta-region-server znode. + */ +message MetaRegionServer { + // The ServerName hosting the meta region currently, or destination server, + // if meta region is in transition. + required ServerName server = 1; + // The major version of the rpc the server speaks. This is used so that + // clients connecting to the cluster can have prior knowledge of what version + // to send to a RegionServer. AsyncHBase will use this to detect versions. + optional uint32 rpc_version = 2; + + // State of the region transition. OPEN means fully operational 'hbase:meta' + optional RegionState.State state = 3; +} + +/** + * Content of the master znode. + */ +message Master { + // The ServerName of the current Master + required ServerName master = 1; + // Major RPC version so that clients can know what version the master can accept. + optional uint32 rpc_version = 2; + optional uint32 info_port = 3; +} + +/** + * Content of the '/hbase/running', cluster state, znode. + */ +message ClusterUp { + // If this znode is present, cluster is up. Currently + // the data is cluster start_date. + required string start_date = 1; +} + +/** + * WAL SplitLog directory znodes have this for content. Used doing distributed + * WAL splitting. Holds current state and name of server that originated split. + */ +message SplitLogTask { + enum State { + UNASSIGNED = 0; + OWNED = 1; + RESIGNED = 2; + DONE = 3; + ERR = 4; + } + enum RecoveryMode { + UNKNOWN = 0; + LOG_SPLITTING = 1; + LOG_REPLAY = 2; + } + required State state = 1; + required ServerName server_name = 2; + optional RecoveryMode mode = 3 [default = UNKNOWN]; +} + +/** + * The znode that holds state of table. + * Deprected, table state is stored in table descriptor on HDFS. + */ +message DeprecatedTableState { + // Table's current state + enum State { + ENABLED = 0; + DISABLED = 1; + DISABLING = 2; + ENABLING = 3; + } + // This is the table's state. If no znode for a table, + // its state is presumed enabled. See o.a.h.h.zookeeper.ZKTable class + // for more. + required State state = 1 [default = ENABLED]; +} + +/** + * Used by replication. Holds a replication peer key. + */ +message ReplicationPeer { + // clusterkey is the concatenation of the slave cluster's + // hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent + required string clusterkey = 1; + optional string replicationEndpointImpl = 2; + repeated BytesBytesPair data = 3; + repeated NameStringPair configuration = 4; +} + +/** + * Used by replication. Holds whether enabled or disabled + */ +message ReplicationState { + enum State { + ENABLED = 0; + DISABLED = 1; + } + required State state = 1; +} + +/** + * Used by replication. Holds the current position in an WAL file. + */ +message ReplicationHLogPosition { + required int64 position = 1; +} + +/** + * Used by replication. Used to lock a region server during failover. + */ +message ReplicationLock { + required string lock_owner = 1; +} + +/** + * Metadata associated with a table lock in zookeeper + */ +message TableLock { + optional TableName table_name = 1; + optional ServerName lock_owner = 2; + optional int64 thread_id = 3; + optional bool is_shared = 4; + optional string purpose = 5; + optional int64 create_time = 6; +} diff --git a/hbase-native-client/rpc/CMakeLists.txt b/hbase-native-client/rpc/CMakeLists.txt deleted file mode 100644 index 2456923..0000000 --- a/hbase-native-client/rpc/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - diff --git a/hbase-native-client/third-party/BUCK b/hbase-native-client/third-party/BUCK index 89d2c93..b7baa86 100644 --- a/hbase-native-client/third-party/BUCK +++ b/hbase-native-client/third-party/BUCK @@ -21,7 +21,8 @@ def add_system_libs(names = []): prebuilt_cxx_library( name = name, lib_name = name, - lib_dir = "/usr/lib/x86_64-linux-gnu" + lib_dir = "/usr/lib/x86_64-linux-gnu", + visibility = [ 'PUBLIC', ], ) rules.append(":" + name) @@ -30,6 +31,7 @@ def add_system_libs(names = []): system_libs = [ "double-conversion", "glog", + "protobuf", "gflags", "unwind", "lzma", -- 2.6.3