From 40f1e6458067b3b63789cd117befb7edc12cb8de Mon Sep 17 00:00:00 2001 From: Elliott Clark Date: Mon, 4 Apr 2016 13:52:05 -0700 Subject: [PATCH] HBASE-14855 Connect to regionserver --- hbase-native-client/core/BUCK | 27 +++++- hbase-native-client/core/client-dispatcher.cc | 53 ++++++++++ hbase-native-client/core/client-dispatcher.h | 42 ++++++++ .../core/client-serialize-handler.h | 107 +++++++++++++++++++++ hbase-native-client/core/client.cc | 16 ++- hbase-native-client/core/client.h | 16 ++- hbase-native-client/core/connection-factory.cc | 56 +++++++++++ hbase-native-client/core/connection-factory.h | 39 ++++++++ hbase-native-client/core/get-request.cc | 19 ++++ hbase-native-client/core/get-request.h | 34 +++++++ hbase-native-client/core/get-result.cc | 19 ++++ hbase-native-client/core/get-result.h | 32 ++++++ hbase-native-client/core/location-cache-test.cc | 25 ++++- hbase-native-client/core/location-cache.cc | 52 +++++++++- hbase-native-client/core/location-cache.h | 18 ++++ hbase-native-client/core/native-client-test-env.cc | 6 +- hbase-native-client/core/pipeline.cc | 43 +++++++++ hbase-native-client/core/pipeline.h | 34 +++++++ hbase-native-client/core/request.h | 33 +++++++ hbase-native-client/core/response.h | 34 +++++++ hbase-native-client/core/service.h | 26 +++++ hbase-native-client/core/simple-client.cc | 59 ++++++++++++ .../core/simple-native-client-test.cc | 5 +- hbase-native-client/core/table-name.cc | 19 ++++ hbase-native-client/core/table-name.h | 32 ++++++ hbase-native-client/if/BUCK | 24 ++++- hbase-native-client/third-party/BUCK | 105 ++++++++++---------- 27 files changed, 891 insertions(+), 84 deletions(-) create mode 100644 hbase-native-client/core/client-dispatcher.cc create mode 100644 hbase-native-client/core/client-dispatcher.h create mode 100644 hbase-native-client/core/client-serialize-handler.h create mode 100644 hbase-native-client/core/connection-factory.cc create mode 100644 hbase-native-client/core/connection-factory.h create mode 100644 hbase-native-client/core/get-request.cc create mode 100644 hbase-native-client/core/get-request.h create mode 100644 hbase-native-client/core/get-result.cc create mode 100644 hbase-native-client/core/get-result.h create mode 100644 hbase-native-client/core/pipeline.cc create mode 100644 hbase-native-client/core/pipeline.h create mode 100644 hbase-native-client/core/request.h create mode 100644 hbase-native-client/core/response.h create mode 100644 hbase-native-client/core/service.h create mode 100644 hbase-native-client/core/simple-client.cc create mode 100644 hbase-native-client/core/table-name.cc create mode 100644 hbase-native-client/core/table-name.h diff --git a/hbase-native-client/core/BUCK b/hbase-native-client/core/BUCK index d1e89d1..77e3913 100644 --- a/hbase-native-client/core/BUCK +++ b/hbase-native-client/core/BUCK @@ -16,29 +16,45 @@ # limitations under the License. cxx_library(name="core", - headers=[ + exported_headers=[ "admin.h", "client.h", + "client-dispatcher.h", + "client-serialize-handler.h", "connection.h", + "connection-factory.h", "connection_attr.h", "delete.h", + "get-request.h", + "get-result.h", "get.h", "hbase_macros.h", + "location-cache.h", "mutation.h", + "pipeline.h", "put.h", + "request.h", + "response.h", "scanner.h", - "location-cache.h", + "service.h", + "table-name.h", ], srcs=[ "admin.cc", "client.cc", + "client-dispatcher.cc", "connection.cc", + "connection-factory.cc", + "delete.cc", + "get-request.cc", + "get-result.cc", "get.cc", + "location-cache.cc", "mutation.cc", + "pipeline.cc", "put.cc", - "delete.cc", "scanner.cc", - "location-cache.cc", + "table-name.cc", ], deps=[ "//if:if", @@ -68,3 +84,6 @@ cxx_test(name="location-cache-test", ":core", ], run_test_separately=True, ) +cxx_binary(name="simple-client", + srcs=["simple-client.cc", ], + deps=[":core", ], ) diff --git a/hbase-native-client/core/client-dispatcher.cc b/hbase-native-client/core/client-dispatcher.cc new file mode 100644 index 0000000..ed16718 --- /dev/null +++ b/hbase-native-client/core/client-dispatcher.cc @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +#include "client-dispatcher.h" + +using namespace folly; +using namespace hbase; +using namespace wangle; + +void ClientDispatcher::read(Context *ctx, Response in) { + auto call_id = in.call_id(); + auto search = requests_.find(call_id); + CHECK(search != requests_.end()); + auto p = std::move(search->second); + requests_.erase(call_id); + + // TODO: check if the response is an exception. If it is then set that. + p.setValue(in); +} + +Future ClientDispatcher::operator()(Request arg) { + auto call_id = current_call_id_++; + arg.set_call_id(call_id); + auto &p = requests_[call_id]; + auto f = p.getFuture(); + p.setInterruptHandler([call_id, this](const folly::exception_wrapper &e) { + this->requests_.erase(call_id); + }); + this->pipeline_->write(arg); + + return f; +} + +Future ClientDispatcher::close() { return ClientDispatcherBase::close(); } + +Future ClientDispatcher::close(Context *ctx) { + return ClientDispatcherBase::close(ctx); +} diff --git a/hbase-native-client/core/client-dispatcher.h b/hbase-native-client/core/client-dispatcher.h new file mode 100644 index 0000000..2f0cdc4 --- /dev/null +++ b/hbase-native-client/core/client-dispatcher.h @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#pragma once + +#include + +#include "pipeline.h" +#include "request.h" +#include "response.h" + +namespace hbase { +class ClientDispatcher + : public wangle::ClientDispatcherBase { +public: + void read(Context *ctx, Response in) override; + folly::Future operator()(Request arg) override; + virtual folly::Future close(Context *ctx) override; + virtual folly::Future close() override; + +private: + std::unordered_map> requests_; + uint32_t current_call_id_ = 1; +}; +} // hbase diff --git a/hbase-native-client/core/client-serialize-handler.h b/hbase-native-client/core/client-serialize-handler.h new file mode 100644 index 0000000..d5dab60 --- /dev/null +++ b/hbase-native-client/core/client-serialize-handler.h @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +#pragma once + +#include + +#include "if/HBase.pb.h" +#include "if/RPC.pb.h" +#include "request.h" +#include "response.h" + +namespace hbase { +const static std::string PREAMBLE = "HBas"; +class ClientSerializeHandler + : public wangle::Handler, Response, Request, + std::unique_ptr> { +public: + // TODO: Make this actually do ANYTHING. + virtual void read(Context *ctx, std::unique_ptr msg) override { + Response received; + ctx->fireRead(received); + } + + virtual folly::Future write(Context *ctx, Request r) override { + // Keep track of if we have sent the header. + if (need_send_header_) { + need_send_header_ = false; + write_preamble(ctx); + write_header(ctx); + } + + // Send out the actual request and not just a test string. + std::string out{"test"}; + return ctx->fireWrite(prepend_length(folly::IOBuf::copyBuffer(out))); + } + + folly::Future write_preamble(Context *ctx) { + auto magic = folly::IOBuf::copyBuffer(PREAMBLE); + auto buf = folly::IOBuf::create(2); + buf->append(2); + folly::io::RWPrivateCursor c(buf.get()); + + // Version + c.write((uint8_t)0); + // Standard security aka Please don't lie to me. + c.write((uint8_t)80); + magic->appendChain(std::move(buf)); + return ctx->fireWrite(std::move(magic)); + } + + folly::Future write_header(Context *ctx) { + pb::ConnectionHeader h; + + // TODO: Make this not a total lie. + h.mutable_user_info()->set_effective_user("elliott"); + // The service name that we want to talk to. + // + // Right now we're completely ignoring the service interface. + // That may or may not be the correct thing to do. + // It worked for a while with the java client; until it + // didn't. + h.set_service_name("ClientService"); + // TODO: Make this 1 copy. + auto msg = folly::IOBuf::copyBuffer(h.SerializeAsString()); + return ctx->fireWrite(prepend_length(std::move(msg))); + } + + // Our own simple version of LengthFieldPrepender + std::unique_ptr + prepend_length(std::unique_ptr msg) { + // Java ints are 4 long. So create a buffer that large + auto len_buf = folly::IOBuf::create(4); + // Then make those bytes visible. + len_buf->append(4); + + folly::io::RWPrivateCursor c(len_buf.get()); + // Get the size of the data to be pushed out the network. + auto size = msg->computeChainDataLength(); + + // Write the length to this IOBuf. + c.writeBE((uint32_t)size); + + // Then attach the origional to the back of len_buf + len_buf->appendChain(std::move(msg)); + return len_buf; + } + +private: + bool need_send_header_ = true; +}; +} diff --git a/hbase-native-client/core/client.cc b/hbase-native-client/core/client.cc index a04daee..5e9e7bb 100644 --- a/hbase-native-client/core/client.cc +++ b/hbase-native-client/core/client.cc @@ -24,19 +24,17 @@ #include #include +#include + #include "if/ZooKeeper.pb.h" +#include "core/connection-factory.h" using namespace folly; +using namespace std; using namespace hbase::pb; -int main(int argc, char *argv[]) { - MetaRegionServer mrs; - google::ParseCommandLineFlags(&argc, &argv, true); - google::InitGoogleLogging(argv[0]); +namespace hbase { - FB_LOG_EVERY_MS(INFO, 10000) << "Hello"; - for (long i = 0; i < 10000000; i++) { - FB_LOG_EVERY_MS(INFO, 1) << Random::rand32(); - } - return 0; +Client::Client(string quorum_spec) + : location_cache(quorum_spec, wangle::getCPUExecutor()) {} } diff --git a/hbase-native-client/core/client.h b/hbase-native-client/core/client.h index 35a3bd8..f386d3b 100644 --- a/hbase-native-client/core/client.h +++ b/hbase-native-client/core/client.h @@ -20,7 +20,21 @@ #pragma once #include +#include #include "if/Cell.pb.h" -class Client {}; +#include "get-request.h" +#include "get-result.h" + +namespace hbase { +class Client { +public: + explicit Client(std::string quorum_spec); + folly::Future get(const GetRequest &get_request); + +private: + LocationCache location_cache; +}; + +} /* hbase */ diff --git a/hbase-native-client/core/connection-factory.cc b/hbase-native-client/core/connection-factory.cc new file mode 100644 index 0000000..3b21edc --- /dev/null +++ b/hbase-native-client/core/connection-factory.cc @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "connection-factory.h" + +#include +#include +#include +#include +#include +#include + +#include "client-dispatcher.h" +#include "pipeline.h" +#include "request.h" +#include "response.h" +#include "service.h" + +using namespace folly; +using namespace hbase; +using namespace wangle; + +ConnectionFactory::ConnectionFactory() { + bootstrap_.group(std::make_shared(2)); + bootstrap_.pipelineFactory(std::make_shared()); +} + +Future ConnectionFactory::make_connection(std::string host, + int port) { + + // Connect to a given server + // Then when connected create a ClientDispactcher. + auto srv = bootstrap_.connect(SocketAddress(host, port, true)) + .then([](SerializePipeline *pipeline) { + ClientDispatcher dispatcher; + dispatcher.setPipeline(pipeline); + return dispatcher; + }); + return srv; +} diff --git a/hbase-native-client/core/connection-factory.h b/hbase-native-client/core/connection-factory.h new file mode 100644 index 0000000..f256dea --- /dev/null +++ b/hbase-native-client/core/connection-factory.h @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +#pragma once + +#include +#include + +#include "service.h" +#include "pipeline.h" +#include "client-dispatcher.h" +#include "request.h" +#include "response.h" + +namespace hbase { +class ConnectionFactory { +public: + ConnectionFactory(); + folly::Future make_connection(std::string host, int port); + +private: + wangle::ClientBootstrap bootstrap_; +}; +} // hbase diff --git a/hbase-native-client/core/get-request.cc b/hbase-native-client/core/get-request.cc new file mode 100644 index 0000000..fe86080 --- /dev/null +++ b/hbase-native-client/core/get-request.cc @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +#include "get-request.h" diff --git a/hbase-native-client/core/get-request.h b/hbase-native-client/core/get-request.h new file mode 100644 index 0000000..3fbb3cd --- /dev/null +++ b/hbase-native-client/core/get-request.h @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +#pragma once + +#include +#include "table-name.h" + +namespace hbase { + +class GetRequest { +public: + GetRequest(TableName table_name, std::string key); + +private: + TableName table_name_; + std::string key_; +}; +} diff --git a/hbase-native-client/core/get-result.cc b/hbase-native-client/core/get-result.cc new file mode 100644 index 0000000..4c979c4 --- /dev/null +++ b/hbase-native-client/core/get-result.cc @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +#include "get-result.h" diff --git a/hbase-native-client/core/get-result.h b/hbase-native-client/core/get-result.h new file mode 100644 index 0000000..8fe1bc5 --- /dev/null +++ b/hbase-native-client/core/get-result.h @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +#pragma once + +#include + +namespace hbase { + +class GetResult { +public: + explicit GetResult(std::string key); + +private: + std::string key_; +}; +} diff --git a/hbase-native-client/core/location-cache-test.cc b/hbase-native-client/core/location-cache-test.cc index 3106e36..70ca6f1 100644 --- a/hbase-native-client/core/location-cache-test.cc +++ b/hbase-native-client/core/location-cache-test.cc @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ #include #include #include @@ -8,7 +26,8 @@ using namespace hbase; TEST(LocationCacheTest, TestGetMetaNodeContents) { // TODO(elliott): need to make a test utility for this. LocationCache cache{"localhost:2181", wangle::getCPUExecutor()}; - auto result = cache.LocateMeta(); - result.wait(); - ASSERT_FALSE(result.hasException()); + auto f = cache.LocateMeta(); + auto result = f.get(); + ASSERT_FALSE(f.hasException()); + ASSERT_TRUE(result.has_port()); } diff --git a/hbase-native-client/core/location-cache.cc b/hbase-native-client/core/location-cache.cc index cf61e24..50e911f 100644 --- a/hbase-native-client/core/location-cache.cc +++ b/hbase-native-client/core/location-cache.cc @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ #include "location-cache.h" #include @@ -50,18 +68,46 @@ void LocationCache::RefreshMetaLocation() { ServerName LocationCache::ReadMetaLocation() { char contents[4096]; + // This needs to be int rather than size_t as that's what ZK expects. int len = sizeof(contents); // TODO(elliott): handle disconnects/reconntion as needed. int zk_result = zoo_get(this->zk_, META_LOCATION.c_str(), 0, contents, &len, nullptr); - - if (zk_result != ZOK) { + if (zk_result != ZOK || len < 9) { LOG(ERROR) << "Error getting meta location."; throw runtime_error("Error getting meta location"); } + // There should be a magic number for recoverable zk + if (static_cast(contents[0]) != 255) { + LOG(ERROR) << "Magic number not in ZK znode data expected 255 got =" + << unsigned(static_cast(contents[0])); + throw runtime_error("Magic number not in znode data"); + } + // pos will keep track of skipped bytes. + int pos = 1; + // How long is the id? + int id_len = 0; + for (int i = 0; i < 4; i++) { + id_len = id_len << 8; + id_len = id_len | static_cast(contents[pos]); + pos++; + } + // Skip the id + pos += id_len; + // Then all protobuf's for HBase are prefixed with a magic string. + // PBUF, so we skip that. + // TODO(eclark): check to make sure that the magic string is correct + // though I am not sure that will get us much. + pos += 4; MetaRegionServer mrs; - mrs.ParseFromArray(contents, len); + // Try to decode the protobuf. + // If there's an error bail out. + if (mrs.ParseFromArray(contents + pos, len - pos) == false) { + LOG(ERROR) << "Error parsing Protobuf Message"; + throw runtime_error("Error parsing protobuf"); + } + return mrs.server(); } } diff --git a/hbase-native-client/core/location-cache.h b/hbase-native-client/core/location-cache.h index 8dc2760..bf9a7a0 100644 --- a/hbase-native-client/core/location-cache.h +++ b/hbase-native-client/core/location-cache.h @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ #pragma once #include diff --git a/hbase-native-client/core/native-client-test-env.cc b/hbase-native-client/core/native-client-test-env.cc index 07f30a6..0269a43 100644 --- a/hbase-native-client/core/native-client-test-env.cc +++ b/hbase-native-client/core/native-client-test-env.cc @@ -22,7 +22,7 @@ namespace { class NativeClientTestEnv : public ::testing::Environment { - public: +public: void SetUp() override { // start local HBase cluster to be reused by all tests auto result = system("bin/start_local_hbase_and_wait.sh"); @@ -36,9 +36,9 @@ class NativeClientTestEnv : public ::testing::Environment { } }; -} // anonymous +} // anonymous -int main(int argc, char** argv) { +int main(int argc, char **argv) { testing::InitGoogleTest(&argc, argv); ::testing::AddGlobalTestEnvironment(new NativeClientTestEnv()); return RUN_ALL_TESTS(); diff --git a/hbase-native-client/core/pipeline.cc b/hbase-native-client/core/pipeline.cc new file mode 100644 index 0000000..b309527 --- /dev/null +++ b/hbase-native-client/core/pipeline.cc @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +#include "pipeline.h" + +#include +#include +#include +#include +#include +#include + +#include "client-serialize-handler.h" + +using namespace folly; +using namespace hbase; +using namespace wangle; + +SerializePipeline::Ptr +RpcPipelineFactory::newPipeline(std::shared_ptr sock) { + auto pipeline = SerializePipeline::create(); + pipeline->addBack(AsyncSocketHandler(sock)); + pipeline->addBack(EventBaseHandler()); + pipeline->addBack(LengthFieldBasedFrameDecoder()); + pipeline->addBack(ClientSerializeHandler()); + pipeline->finalize(); + return pipeline; +} diff --git a/hbase-native-client/core/pipeline.h b/hbase-native-client/core/pipeline.h new file mode 100644 index 0000000..7c87aaa --- /dev/null +++ b/hbase-native-client/core/pipeline.h @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +#pragma once + +#include +#include +#include "request.h" +#include "response.h" + +namespace hbase { +using SerializePipeline = wangle::Pipeline; + +class RpcPipelineFactory : public wangle::PipelineFactory { +public: + SerializePipeline::Ptr + newPipeline(std::shared_ptr sock) override; +}; +} // hbase diff --git a/hbase-native-client/core/request.h b/hbase-native-client/core/request.h new file mode 100644 index 0000000..4246467 --- /dev/null +++ b/hbase-native-client/core/request.h @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +#pragma once + +#include + +namespace hbase { +class Request { +public: + Request() : call_id_(0) {} + uint32_t call_id() { return call_id_; } + void set_call_id(uint32_t call_id) { call_id_ = call_id; } + +private: + uint32_t call_id_; +}; +} diff --git a/hbase-native-client/core/response.h b/hbase-native-client/core/response.h new file mode 100644 index 0000000..f92abe2 --- /dev/null +++ b/hbase-native-client/core/response.h @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +#pragma once + +#include + +namespace hbase { + +class Response { +public: + Response() : call_id_(0) {} + uint32_t call_id() { return call_id_; } + void set_call_id(uint32_t call_id) { call_id_ = call_id; } + +private: + uint32_t call_id_; +}; +} diff --git a/hbase-native-client/core/service.h b/hbase-native-client/core/service.h new file mode 100644 index 0000000..a82b4af --- /dev/null +++ b/hbase-native-client/core/service.h @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +#pragma once + +#include "request.h" +#include "response.h" + +namespace hbase { +using HBaseService = wangle::Service; +} diff --git a/hbase-native-client/core/simple-client.cc b/hbase-native-client/core/simple-client.cc new file mode 100644 index 0000000..81992a9 --- /dev/null +++ b/hbase-native-client/core/simple-client.cc @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include + +#include +#include +#include +#include +#include + +#include "core/client.h" +#include "core/connection-factory.h" +#include "if/ZooKeeper.pb.h" + +using namespace folly; +using namespace std; +using namespace hbase::pb; + +int main(int argc, char *argv[]) { + google::ParseCommandLineFlags(&argc, &argv, true); + google::InitGoogleLogging(argv[0]); + + // Show that we can create a client + // hbase::Client client{"localhost:2181"}; + + // Create a connection factory + hbase::ConnectionFactory cf; + + hbase::LocationCache cache{"localhost:2181", wangle::getCPUExecutor()}; + + auto result = cache.LocateMeta().get(); + cout << "ServerName = " << result.host_name() << ":" << result.port() << endl; + + // Create a connection to the local host + auto conn = cf.make_connection(result.host_name(), result.port()).get(); + + // Send the request + hbase::Request r; + conn(r).get(); + + return 0; +} diff --git a/hbase-native-client/core/simple-native-client-test.cc b/hbase-native-client/core/simple-native-client-test.cc index ef564f7..ee39986 100644 --- a/hbase-native-client/core/simple-native-client-test.cc +++ b/hbase-native-client/core/simple-native-client-test.cc @@ -22,7 +22,4 @@ /** * Sample test. */ -TEST(SampleTest, sample) { - EXPECT_TRUE(true); -} - +TEST(SampleTest, sample) { EXPECT_TRUE(true); } diff --git a/hbase-native-client/core/table-name.cc b/hbase-native-client/core/table-name.cc new file mode 100644 index 0000000..3a528f2 --- /dev/null +++ b/hbase-native-client/core/table-name.cc @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +#include "table-name.h" diff --git a/hbase-native-client/core/table-name.h b/hbase-native-client/core/table-name.h new file mode 100644 index 0000000..0fcf327 --- /dev/null +++ b/hbase-native-client/core/table-name.h @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +#pragma once + +#include + +#include "location-cache.h" +namespace hbase { + +// This is the core class of a HBase client. +class TableName { +public: + explicit TableName(std::string tableName); + explicit TableName(std::string namespaceName, std::string tableName); +}; +} // hbase diff --git a/hbase-native-client/if/BUCK b/hbase-native-client/if/BUCK index 9b989b5..5ff617d 100644 --- a/hbase-native-client/if/BUCK +++ b/hbase-native-client/if/BUCK @@ -1,3 +1,21 @@ +## +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + PROTO_SRCS = glob(['*.proto']) HEADER_FILENAMES = [ x.replace('.proto','.pb.h') for x in PROTO_SRCS] CC_FILENAMES = [ x.replace('.proto', '.pb.cc') for x in PROTO_SRCS] @@ -20,7 +38,7 @@ for cc_filename in CC_FILENAMES: genrule( name = cc_filename, cmd = 'mkdir -p `dirname $OUT` ' - ' && cp $(location :generate-proto-sources)/{} $OUT ' + ' && cp $(location :generate-proto-sources)/*.cc `dirname $OUT` ' ' && cp $(location :generate-proto-sources)/*.h `dirname $OUT`'.format(cc_filename), out = cc_filename, ) @@ -29,9 +47,7 @@ cxx_library( name = 'if', exported_headers = [':' + x for x in HEADER_FILENAMES], srcs = [':' + x for x in CC_FILENAMES], - deps = [ '//third-party:protobuf'] - + [':' + x for x in CC_FILENAMES] - + [ ':' + x for x in HEADER_FILENAMES ], + deps = [ '//third-party:protobuf'], visibility = [ 'PUBLIC', ], exported_deps = ['//third-party:protobuf'] ) diff --git a/hbase-native-client/third-party/BUCK b/hbase-native-client/third-party/BUCK index 6548695..4327530 100644 --- a/hbase-native-client/third-party/BUCK +++ b/hbase-native-client/third-party/BUCK @@ -23,19 +23,30 @@ def add_system_libs(names=[], exported_linker_flags=[]): rules = [] for name in names: + rule_visibility = ['PUBLIC'] gen_rule_name = "gen_lib{}".format(name) - genrule(name=gen_rule_name, - out=gen_rule_name, - bash="mkdir -p $OUT && cp {}/lib{}.a $OUT".format(lib_dir, - name), ) + genrule( + name=gen_rule_name, + out=gen_rule_name, + bash="mkdir -p $OUT && cp {}/lib{}.a $OUT".format(lib_dir, name), ) prebuilt_cxx_library(name=name, lib_name=name, lib_dir='$(location :{})'.format(gen_rule_name), deps=deps, - force_static = True, + force_static=True, exported_deps=exported_deps, - visibility=['PUBLIC'], - exported_linker_flags=exported_linker_flags, ) + visibility=rule_visibility, ) + rules.append(":" + name) + return rules + + +def add_dynamic_libs(names=[]): + rules = [] + for name in names: + prebuilt_cxx_library(name=name, + header_only=True, + exported_linker_flags=["-l" + name], + visibility=["PUBLIC"], ) rules.append(":" + name) return rules @@ -54,58 +65,46 @@ local_libs = [ "glog", "protobuf", ] +dynamic_libs = ["stdc++", "pthread", "ssl", "crypto", "dl", "atomic", ] +dynamic_rules = add_dynamic_libs(dynamic_libs) +tp_dep_rules = add_system_libs(system_libs,) \ + + add_system_libs(local_libs, lib_dir = "/usr/local/lib") \ + + dynamic_rules - - -tp_dep_rules = add_system_libs(system_libs) \ - + add_system_libs(local_libs, lib_dir = "/usr/local/lib") - -zookeeper = add_system_libs(["zookeeper_mt"], lib_dir = "/usr/local/lib") +zookeeper = add_system_libs(["zookeeper_mt"], lib_dir="/usr/local/lib") folly = add_system_libs(['folly'], lib_dir='/usr/local/lib', - exported_deps=tp_dep_rules, - exported_linker_flags=["-pthread", - "-lstdc++", ]) + exported_deps=tp_dep_rules, ) folly_bench = add_system_libs(['follybenchmark'], lib_dir='/usr/local/lib', - exported_deps=tp_dep_rules + folly, - exported_linker_flags=["-pthread", - "-lstdc++", ]) + exported_deps=folly + tp_dep_rules, ) wangle = add_system_libs(['wangle'], lib_dir='/usr/local/lib', - exported_deps=tp_dep_rules + folly, - exported_linker_flags=["-pthread", - "-lstdc++", ]) - + exported_deps=folly + tp_dep_rules) genrule( -name = "gen_zk", -out = "gen_zk", -bash = "mkdir -p $OUT && wget http://www-us.apache.org/dist/zookeeper/zookeeper-3.4.8/zookeeper-3.4.8.tar.gz && tar zxf zookeeper-3.4.8.tar.gz && rm -rf zookeeper-3.4.8.tar.gz && cd zookeeper-3.4.8 && cd src/c && ./configure --prefix=$OUT && make && make install && cd $OUT && rm -rf zookeeper-3.4.8*" -) -cxx_library( - name = 'google-test', - srcs = [ - 'googletest/googletest/src/gtest-all.cc', - 'googletest/googlemock/src/gmock-all.cc', - 'googletest/googlemock/src/gmock_main.cc', - ], - header_namespace = '', - exported_headers = subdir_glob([ - ('googletest/googletest/include', '**/*.h'), - ('googletest/googlemock/include', '**/*.h'), - ]), - headers = subdir_glob([ - ('googletest/googletest', 'src/*.h'), - ('googletest/googletest', 'src/*.cc'), - ('googletest/googlemock', 'src/*.h'), - ('googletest/googlemock', 'src/*.cc'), - ]), - exported_linker_flags = [ - "-pthread", - "-lstdc++", - ], - visibility = [ - 'PUBLIC', - ], -) + name="gen_zk", + out="gen_zk", + bash= + "mkdir -p $OUT && wget http://www-us.apache.org/dist/zookeeper/zookeeper-3.4.8/zookeeper-3.4.8.tar.gz && tar zxf zookeeper-3.4.8.tar.gz && rm -rf zookeeper-3.4.8.tar.gz && cd zookeeper-3.4.8 && cd src/c && ./configure --prefix=$OUT && make && make install && cd $OUT && rm -rf zookeeper-3.4.8*") +cxx_library(name='google-test', + srcs=[ + 'googletest/googletest/src/gtest-all.cc', + 'googletest/googlemock/src/gmock-all.cc', + 'googletest/googlemock/src/gmock_main.cc', + ], + header_namespace='', + exported_headers=subdir_glob([ + ('googletest/googletest/include', '**/*.h'), + ('googletest/googlemock/include', '**/*.h'), + ]), + headers=subdir_glob([ + ('googletest/googletest', 'src/*.h'), + ('googletest/googletest', 'src/*.cc'), + ('googletest/googlemock', 'src/*.h'), + ('googletest/googlemock', 'src/*.cc'), + ]), + exported_deps=dynamic_rules, + visibility=[ + 'PUBLIC', + ], ) -- 2.8.0-rc2