diff --git hbase-native-client/connection/BUCK hbase-native-client/connection/BUCK index bc05be0..19536d5 100644 --- hbase-native-client/connection/BUCK +++ hbase-native-client/connection/BUCK @@ -50,14 +50,8 @@ cxx_library( "//third-party:wangle", ], compiler_flags=['-Weffc++'], - visibility=[ - '//core/...', - ],) + visibility=['//core/...',],) cxx_test( name="connection-pool-test", - srcs=[ - "connection-pool-test.cc", - ], - deps=[ - ":connection", - ],) + srcs=["connection-pool-test.cc",], + deps=[":connection",],) diff --git hbase-native-client/core/BUCK hbase-native-client/core/BUCK index e40236f..13e8361 100644 --- hbase-native-client/core/BUCK +++ hbase-native-client/core/BUCK @@ -43,6 +43,7 @@ cxx_library( "async-rpc-retrying-caller-factory.h", "async-rpc-retrying-caller.h", "hbase-rpc-controller.h", + "zk-util.h", ], srcs=[ "cell.cc", @@ -59,6 +60,7 @@ cxx_library( "request-converter.cc", "response-converter.cc", "table.cc", + "zk-util.cc", ], deps=[ "//exceptions:exceptions", @@ -71,14 +73,10 @@ cxx_library( "//third-party:zookeeper_mt", ], compiler_flags=['-Weffc++', '-ggdb'], - visibility=[ - 'PUBLIC', - ],) + visibility=['PUBLIC',],) cxx_test( name="location-cache-test", - srcs=[ - "location-cache-test.cc", - ], + srcs=["location-cache-test.cc",], deps=[ ":core", "//test-util:test-util", @@ -86,18 +84,12 @@ cxx_test( run_test_separately=True,) cxx_test( name="cell-test", - srcs=[ - "cell-test.cc", - ], - deps=[ - ":core", - ], + srcs=["cell-test.cc",], + deps=[":core",], run_test_separately=True,) cxx_test( name="filter-test", - srcs=[ - "filter-test.cc", - ], + srcs=["filter-test.cc",], deps=[ ":core", "//if:if", @@ -155,9 +147,7 @@ cxx_test( run_test_separately=True,) cxx_test( name="client-test", - srcs=[ - "client-test.cc", - ], + srcs=["client-test.cc",], deps=[ ":core", "//if:if", @@ -165,9 +155,12 @@ cxx_test( "//test-util:test-util", ], run_test_separately=True,) +cxx_test( + name="zk-util-test", + srcs=["zk-util-test.cc",], + deps=[":core",], + run_test_separately=True,) cxx_binary( name="simple-client", - srcs=[ - "simple-client.cc", - ], + srcs=["simple-client.cc",], deps=[":core", "//connection:connection"],) diff --git hbase-native-client/core/async-rpc-retrying-test.cc hbase-native-client/core/async-rpc-retrying-test.cc index f2a79a1..81d726f 100644 --- hbase-native-client/core/async-rpc-retrying-test.cc +++ hbase-native-client/core/async-rpc-retrying-test.cc @@ -143,11 +143,10 @@ class MockRawAsyncTableImpl { }; TEST(AsyncRpcRetryTest, TestGetBasic) { - // Remove already configured env if present. - unsetenv("HBASE_CONF"); - // Using TestUtil to populate test data - hbase::TestUtil* test_util = new hbase::TestUtil(); + auto test_util = std::make_unique(); + test_util->StartMiniCluster(2); + test_util->CreateTable("t", "d"); test_util->TablePut("t", "test2", "d", "2", "value2"); test_util->TablePut("t", "test2", "d", "extra", "value for extra"); @@ -231,12 +230,6 @@ TEST(AsyncRpcRetryTest, TestGetBasic) { hbase::Result result = async_caller->Call().get(); - /*Stopping the connection as we are getting segfault due to some folly issue - The connection stays open and we don't want that. - So we are stopping the connection. - We can remove this once we have fixed the folly part */ - delete test_util; - // Test the values, should be same as in put executed on hbase shell ASSERT_TRUE(!result.IsEmpty()) << "Result shouldn't be empty."; EXPECT_EQ("test2", result.Row()); diff --git hbase-native-client/core/client-test.cc hbase-native-client/core/client-test.cc index 6f74e9c..72680c6 100644 --- hbase-native-client/core/client-test.cc +++ hbase-native-client/core/client-test.cc @@ -59,7 +59,8 @@ class ClientTest : public ::testing::Test { static std::unique_ptr test_util; static void SetUpTestCase() { - test_util = std::make_unique(2, ClientTest::kDefHBaseConfPath.c_str()); + test_util = std::make_unique(); + test_util->StartMiniCluster(2); } }; std::unique_ptr ClientTest::test_util = nullptr; @@ -86,9 +87,9 @@ const std::string ClientTest::kHBaseXmlData( "the License.\n " "*/\n-->\n\n\n"); -TEST(Client, EmptyConfigurationPassedToClient) { ASSERT_ANY_THROW(hbase::Client client); } +TEST_F(ClientTest, EmptyConfigurationPassedToClient) { ASSERT_ANY_THROW(hbase::Client client); } -TEST(Client, ConfigurationPassedToClient) { +TEST_F(ClientTest, ConfigurationPassedToClient) { // Remove already configured env if present. unsetenv("HBASE_CONF"); ClientTest::CreateHBaseConfWithEnv(); @@ -101,7 +102,7 @@ TEST(Client, ConfigurationPassedToClient) { client.Close(); } -TEST(Client, DefaultConfiguration) { +TEST_F(ClientTest, DefaultConfiguration) { // Remove already configured env if present. unsetenv("HBASE_CONF"); ClientTest::CreateHBaseConfWithEnv(); @@ -112,10 +113,6 @@ TEST(Client, DefaultConfiguration) { } TEST_F(ClientTest, Get) { - // Remove already configured env if present. - unsetenv("HBASE_CONF"); - ClientTest::CreateHBaseConfWithEnv(); - // Using TestUtil to populate test data ClientTest::test_util->CreateTable("t", "d"); ClientTest::test_util->TablePut("t", "test2", "d", "2", "value2"); @@ -128,12 +125,8 @@ TEST_F(ClientTest, Get) { // Get to be performed on above HBase Table hbase::Get get(row); - // Create Configuration - hbase::HBaseConfigurationLoader loader; - auto conf = loader.LoadDefaultResources(); - // Create a client - hbase::Client client(conf.value()); + hbase::Client client(*ClientTest::test_util->conf()); // Get connection to HBase Table auto table = client.Table(tn); @@ -152,11 +145,7 @@ TEST_F(ClientTest, Get) { client.Close(); } -TEST(Client, GetForNonExistentTable) { - // Remove already configured env if present. - unsetenv("HBASE_CONF"); - ClientTest::CreateHBaseConfWithEnv(); - +TEST_F(ClientTest, GetForNonExistentTable) { // Create TableName and Row to be fetched from HBase auto tn = folly::to("t_not_exists"); auto row = "test2"; @@ -164,12 +153,8 @@ TEST(Client, GetForNonExistentTable) { // Get to be performed on above HBase Table hbase::Get get(row); - // Create Configuration - hbase::HBaseConfigurationLoader loader; - auto conf = loader.LoadDefaultResources(); - // Create a client - hbase::Client client(conf.value()); + hbase::Client client(*ClientTest::test_util->conf()); // Get connection to HBase Table auto table = client.Table(tn); @@ -183,10 +168,6 @@ TEST(Client, GetForNonExistentTable) { } TEST_F(ClientTest, GetForNonExistentRow) { - // Remove already configured env if present. - unsetenv("HBASE_CONF"); - ClientTest::CreateHBaseConfWithEnv(); - // Using TestUtil to populate test data ClientTest::test_util->CreateTable("t_exists", "d"); @@ -197,12 +178,8 @@ TEST_F(ClientTest, GetForNonExistentRow) { // Get to be performed on above HBase Table hbase::Get get(row); - // Create Configuration - hbase::HBaseConfigurationLoader loader; - auto conf = loader.LoadDefaultResources(); - // Create a client - hbase::Client client(conf.value()); + hbase::Client client(*ClientTest::test_util->conf()); // Get connection to HBase Table auto table = client.Table(tn); diff --git hbase-native-client/core/client.cc hbase-native-client/core/client.cc index f0483ef..3f889d4 100644 --- hbase-native-client/core/client.cc +++ hbase-native-client/core/client.cc @@ -41,7 +41,6 @@ Client::Client(const hbase::Configuration &conf) { init(conf); } void Client::init(const hbase::Configuration &conf) { conf_ = std::make_shared(conf); - auto zk_quorum = conf_->Get(kHBaseZookeeperQuorum_, kDefHBaseZookeeperQuorum_); conn_conf_ = std::make_shared(*conf_); // start thread pools diff --git hbase-native-client/core/client.h hbase-native-client/core/client.h index 6e3c889..040bea0 100644 --- hbase-native-client/core/client.h +++ hbase-native-client/core/client.h @@ -70,8 +70,6 @@ class Client { private: /** Constants */ - static constexpr const char* kHBaseZookeeperQuorum_ = "hbase.zookeeper.quorum"; - static constexpr const char* kDefHBaseZookeeperQuorum_ = "localhost:2181"; /** Parameter name for HBase client IO thread pool size. Defaults to num cpus */ static constexpr const char* kClientIoThreadPoolSize = "hbase.client.io.thread.pool.size"; /** Parameter name for HBase client CPU thread pool size. Defaults to (2 * num cpus) */ diff --git hbase-native-client/core/filter-test.cc hbase-native-client/core/filter-test.cc index 6529b5a..40081cc 100644 --- hbase-native-client/core/filter-test.cc +++ hbase-native-client/core/filter-test.cc @@ -41,6 +41,7 @@ class FilterTest : public ::testing::Test { protected: static void SetUpTestCase() { test_util_ = std::make_unique(); + test_util_->StartMiniCluster(2); } static void TearDownTestCase() { test_util_.release(); } @@ -51,7 +52,6 @@ class FilterTest : public ::testing::Test { static std::unique_ptr test_util_; }; - std::unique_ptr FilterTest::test_util_ = nullptr; TEST_F(FilterTest, GetWithColumnPrefixFilter) { diff --git hbase-native-client/core/location-cache-test.cc hbase-native-client/core/location-cache-test.cc index 75a56d1..4159af4 100644 --- hbase-native-client/core/location-cache-test.cc +++ hbase-native-client/core/location-cache-test.cc @@ -34,6 +34,7 @@ class LocationCacheTest : public ::testing::Test { protected: static void SetUpTestCase() { test_util_ = std::make_unique(); + test_util_->StartMiniCluster(2); } static void TearDownTestCase() { test_util_.release(); } @@ -46,7 +47,6 @@ class LocationCacheTest : public ::testing::Test { std::unique_ptr LocationCacheTest::test_util_ = nullptr; - TEST_F(LocationCacheTest, TestGetMetaNodeContents) { auto cpu = std::make_shared(4); auto io = std::make_shared(4); @@ -105,8 +105,9 @@ TEST_F(LocationCacheTest, TestCaching) { ASSERT_EQ(loc, cache.GetCachedLocation(tn_1, row_a)); // test with two regions - std::string empty; - LocationCacheTest::test_util_->CreateTable("t2", "d", "b", empty); + std::vector keys; + keys.push_back("b"); + LocationCacheTest::test_util_->CreateTable("t2", "d", keys); ASSERT_FALSE(cache.IsLocationCached(tn_2, "a")); loc = cache.LocateRegion(tn_2, "a").get(milliseconds(1000)); @@ -121,7 +122,10 @@ TEST_F(LocationCacheTest, TestCaching) { ASSERT_EQ(loc, cache.GetCachedLocation(tn_2, "ba")); // test with three regions - LocationCacheTest::test_util_->CreateTable("t3", "d", "b", "c"); + keys.clear(); + keys.push_back("b"); + keys.push_back("c"); + LocationCacheTest::test_util_->CreateTable("t3", "d", keys); ASSERT_FALSE(cache.IsLocationCached(tn_3, "c")); ASSERT_FALSE(cache.IsLocationCached(tn_3, "ca")); diff --git hbase-native-client/core/location-cache.cc hbase-native-client/core/location-cache.cc index 9550614..505f48c 100644 --- hbase-native-client/core/location-cache.cc +++ hbase-native-client/core/location-cache.cc @@ -18,13 +18,12 @@ */ #include "core/location-cache.h" +#include #include #include #include #include -#include - #include #include "connection/response.h" #include "connection/rpc-connection.h" @@ -34,6 +33,8 @@ #include "serde/server-name.h" #include "serde/zk.h" +#include + using namespace std; using namespace folly; using hbase::RpcConnection; @@ -63,8 +64,8 @@ LocationCache::LocationCache(std::shared_ptr conf, zk_(nullptr), cached_locations_(), locations_lock_() { - quorum_spec_ = conf_->Get(kHBaseZookeeperQuorum_, kDefHBaseZookeeperQuorum_); - zk_ = zookeeper_init(quorum_spec_.c_str(), nullptr, 1000, 0, 0, 0); + zk_quorum_ = ZKUtil::ParseZooKeeperQuorum(*conf_); + zk_ = zookeeper_init(zk_quorum_.c_str(), nullptr, 1000, 0, 0, 0); } LocationCache::~LocationCache() { @@ -100,15 +101,13 @@ ServerName LocationCache::ReadMetaLocation() { // This needs to be int rather than size_t as that's what ZK expects. int len = buf->capacity(); - std::string zk_node = conf_->Get(kHBaseMetaZnodeName_, kDefHBaseMetaZnodeName_); - zk_node += "/" + kHBaseMetaRegionServer_; + std::string zk_node = ZKUtil::MetaZNode(*conf_); // TODO(elliott): handle disconnects/reconntion as needed. int zk_result = zoo_get(this->zk_, zk_node.c_str(), 0, reinterpret_cast(buf->writableData()), &len, nullptr); if (zk_result != ZOK || len < 9) { LOG(ERROR) << "Error getting meta location."; - throw runtime_error("Error getting meta location. Quorum " + - conf_->Get(kHBaseZookeeperQuorum_, "")); + throw runtime_error("Error getting meta location. Quorum: " + zk_quorum_); } buf->append(len); diff --git hbase-native-client/core/location-cache.h hbase-native-client/core/location-cache.h index 4d66e06..c1c5f62 100644 --- hbase-native-client/core/location-cache.h +++ hbase-native-client/core/location-cache.h @@ -36,6 +36,7 @@ #include "core/meta-utils.h" #include "core/region-location.h" #include "serde/table-name.h" +#include "zk-util.h" namespace hbase { // Forward @@ -172,6 +173,8 @@ class LocationCache { */ void ClearCachedLocation(const hbase::pb::TableName &tn, const std::string &row); + const std::string &zk_quorum() { return zk_quorum_; } + private: void RefreshMetaLocation(); hbase::pb::ServerName ReadMetaLocation(); @@ -180,15 +183,9 @@ class LocationCache { const hbase::pb::TableName &tn); std::shared_ptr GetNewTableLocations(const hbase::pb::TableName &tn); - const std::string kHBaseZookeeperQuorum_ = "hbase.zookeeper.quorum"; - const std::string kDefHBaseZookeeperQuorum_ = "localhost:2181"; - const std::string kHBaseMetaZnodeName_ = "zookeeper.znode.parent"; - const std::string kDefHBaseMetaZnodeName_ = "/hbase"; - const std::string kHBaseMetaRegionServer_ = "meta-region-server"; - /* data */ std::shared_ptr conf_; - std::string quorum_spec_; + std::string zk_quorum_; std::shared_ptr cpu_executor_; std::unique_ptr> meta_promise_; std::mutex meta_lock_; diff --git hbase-native-client/core/zk-util-test.cc hbase-native-client/core/zk-util-test.cc new file mode 100644 index 0000000..b0cefdc --- /dev/null +++ hbase-native-client/core/zk-util-test.cc @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +#include + +#include "core/zk-util.h" + +using hbase::Configuration; +using hbase::ZKUtil; + +TEST(ZKUtilTest, ParseZooKeeperQuorum) { + Configuration conf{}; + conf.Set(ZKUtil::kHBaseZookeeperQuorum_, "s1"); + conf.SetInt(ZKUtil::kHBaseZookeeperClientPort_, 100); + + ASSERT_EQ("s1:100", ZKUtil::ParseZooKeeperQuorum(conf)); + + conf.Set(ZKUtil::kHBaseZookeeperQuorum_, "s1:42"); + + ASSERT_EQ("s1:42", ZKUtil::ParseZooKeeperQuorum(conf)); + + conf.Set(ZKUtil::kHBaseZookeeperQuorum_, "s1,s2,s3"); + ASSERT_EQ("s1:100,s2:100,s3:100", ZKUtil::ParseZooKeeperQuorum(conf)); + + conf.Set(ZKUtil::kHBaseZookeeperQuorum_, "s1:42,s2:42,s3:42"); + ASSERT_EQ("s1:42,s2:42,s3:42", ZKUtil::ParseZooKeeperQuorum(conf)); +} + +TEST(ZKUtilTest, MetaZNode) { + Configuration conf{}; + ASSERT_EQ("/hbase/meta-region-server", ZKUtil::MetaZNode(conf)); + + conf.Set(ZKUtil::kHBaseZnodeParent_, "/hbase-secure"); + ASSERT_EQ("/hbase-secure/meta-region-server", ZKUtil::MetaZNode(conf)); +} diff --git hbase-native-client/core/zk-util.cc hbase-native-client/core/zk-util.cc new file mode 100644 index 0000000..50ea92a --- /dev/null +++ hbase-native-client/core/zk-util.cc @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "core/zk-util.h" + +#include +#include + +#include + +namespace hbase { + +/** + * Returns a "proper" zookeeper quorum string, from hbase's broken quorum string formats. In + * hbase.zookeeper.quorum, the ports are not listed explicitly per server (eg. s1,s2,s3), + * however ZooKeeper expects the string of the format s1:2181,s2:2181,s3:2181. This code + * appends the "clientPort" to each node in the quorum string if not there. + */ +std::string ZKUtil::ParseZooKeeperQuorum(const hbase::Configuration& conf) { + auto zk_quorum = conf.Get(kHBaseZookeeperQuorum_, kDefHBaseZookeeperQuorum_); + auto zk_port = conf.GetInt(kHBaseZookeeperClientPort_, kDefHBaseZookeeperClientPort_); + + std::vector zk_quorum_parts; + boost::split(zk_quorum_parts, zk_quorum, boost::is_any_of(","), boost::token_compress_on); + std::vector servers; + for (auto server : zk_quorum_parts) { + if (boost::contains(server, ":")) { + servers.push_back(server); + } else { + servers.push_back(server + ":" + folly::to(zk_port)); + } + } + return boost::join(servers, ","); +} + +std::string ZKUtil::MetaZNode(const hbase::Configuration& conf) { + std::string zk_node = conf.Get(kHBaseZnodeParent_, kDefHBaseZnodeParent_) + "/"; + zk_node += kHBaseMetaRegionServer_; + return zk_node; +} + +} // namespace hbase diff --git hbase-native-client/core/zk-util.h hbase-native-client/core/zk-util.h new file mode 100644 index 0000000..8f2d627 --- /dev/null +++ hbase-native-client/core/zk-util.h @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +#pragma once + +#include +#include +#include "core/configuration.h" + +namespace hbase { + +class ZKUtil { + public: + static constexpr const char* kHBaseZookeeperQuorum_ = "hbase.zookeeper.quorum"; + static constexpr const char* kDefHBaseZookeeperQuorum_ = "localhost:2181"; + static constexpr const char* kHBaseZookeeperClientPort_ = "hbase.zookeeper.property.clientPort"; + static constexpr const int32_t kDefHBaseZookeeperClientPort_ = 2181; + static constexpr const char* kHBaseZnodeParent_ = "zookeeper.znode.parent"; + static constexpr const char* kDefHBaseZnodeParent_ = "/hbase"; + static constexpr const char* kHBaseMetaRegionServer_ = "meta-region-server"; + + static std::string ParseZooKeeperQuorum(const hbase::Configuration& conf); + + static std::string MetaZNode(const hbase::Configuration& conf); +}; +} // namespace hbase diff --git hbase-native-client/security/BUCK hbase-native-client/security/BUCK index d602ff3..e176c90 100644 --- hbase-native-client/security/BUCK +++ hbase-native-client/security/BUCK @@ -19,9 +19,7 @@ # to a single server. cxx_library( name="security", - exported_headers=[ - "user.h", - ], + exported_headers=["user.h",], srcs=[], deps=[], compiler_flags=['-Weffc++'], diff --git hbase-native-client/serde/BUCK hbase-native-client/serde/BUCK index 76d1c72..38e7b4d 100644 --- hbase-native-client/serde/BUCK +++ hbase-native-client/serde/BUCK @@ -41,54 +41,28 @@ cxx_library( ":region-info-deserializer-test", ], compiler_flags=['-Weffc++'], - visibility=[ - 'PUBLIC', - ],) + visibility=['PUBLIC',],) cxx_test( name="table-name-test", - srcs=[ - "table-name-test.cc", - ], - deps=[ - ":serde", - ],) + srcs=["table-name-test.cc",], + deps=[":serde",],) cxx_test( name="server-name-test", - srcs=[ - "server-name-test.cc", - ], - deps=[ - ":serde", - ],) + srcs=["server-name-test.cc",], + deps=[":serde",],) cxx_test( name="client-serializer-test", - srcs=[ - "client-serializer-test.cc", - ], - deps=[ - ":serde", - ],) + srcs=["client-serializer-test.cc",], + deps=[":serde",],) cxx_test( name="client-deserializer-test", - srcs=[ - "client-deserializer-test.cc", - ], - deps=[ - ":serde", - ],) + srcs=["client-deserializer-test.cc",], + deps=[":serde",],) cxx_test( name="zk-deserializer-test", - srcs=[ - "zk-deserializer-test.cc", - ], - deps=[ - ":serde", - ],) + srcs=["zk-deserializer-test.cc",], + deps=[":serde",],) cxx_test( name="region-info-deserializer-test", - srcs=[ - "region-info-deserializer-test.cc", - ], - deps=[ - ":serde", - ],) + srcs=["region-info-deserializer-test.cc",], + deps=[":serde",],) diff --git hbase-native-client/test-util/mini-cluster.cc hbase-native-client/test-util/mini-cluster.cc index 88a091c..fb58fc1 100644 --- hbase-native-client/test-util/mini-cluster.cc +++ hbase-native-client/test-util/mini-cluster.cc @@ -18,13 +18,12 @@ */ #include "test-util/mini-cluster.h" -#include #include -#include +#include #include +#include #include - using hbase::MiniCluster; JNIEnv *MiniCluster::CreateVM(JavaVM **jvm) { @@ -60,7 +59,7 @@ JNIEnv *MiniCluster::CreateVM(JavaVM **jvm) { } fd.close(); } - auto options = std::string {"-Djava.class.path="} + clspath; + auto options = std::string{"-Djava.class.path="} + clspath; jvm_options.optionString = const_cast(options.c_str()); args.options = &jvm_options; args.ignoreUnrecognized = 0; @@ -74,40 +73,11 @@ JNIEnv *MiniCluster::CreateVM(JavaVM **jvm) { return env_; } -void MiniCluster::WriteConf(jobject conf, const std::string& filepath) { - jclass class_fdesc = env_->FindClass("java/io/FileDescriptor"); - // construct a new FileDescriptor - jmethodID const_fdesc = env_->GetMethodID(class_fdesc, "", "()V"); - - jobject file = env_->NewObject(class_fdesc, const_fdesc); - jfieldID field_fd = env_->GetFieldID(class_fdesc, "fd", "I"); - - int fd = open(filepath.c_str(), O_RDWR | O_NONBLOCK | O_CREAT, S_IRWXU); - if (fd < 0) { - LOG(INFO) << "Couldn't open file " << filepath.c_str(); - exit(-1); - } - env_->SetIntField(file, field_fd, fd); - - jclass cls_outstream = env_->FindClass("java/io/FileOutputStream"); - jmethodID ctor_stream = env_->GetMethodID(cls_outstream, "", "(Ljava/io/FileDescriptor;)V"); - if (ctor_stream == NULL) { - LOG(INFO) << "Couldn't get ctor for FileOutputStream"; - exit(-1); - } - jobject file_outstream = env_->NewObject(cls_outstream, ctor_stream, file); - if (file_outstream == NULL) { - LOG(INFO) << "Couldn't create FileOutputStream"; - exit(-1); - } - jmethodID writeXmlMid = env_->GetMethodID(conf_class_, "writeXml", "(Ljava/io/OutputStream;)V"); - env_->CallObjectMethod(conf, writeXmlMid, file_outstream); -} void MiniCluster::Setup() { jmethodID constructor; pthread_mutex_lock(&count_mutex_); if (env_ == NULL) { - env_ = CreateVM(&jvm); + env_ = CreateVM(&jvm_); if (env_ == NULL) { exit(-1); } @@ -119,7 +89,7 @@ void MiniCluster::Setup() { exit(-1); } jmethodID mid = env_->GetStaticMethodID(testing_util_class_, "createLocalHTU", - "()Lorg/apache/hadoop/hbase/HBaseTestingUtility;"); + "()Lorg/apache/hadoop/hbase/HBaseTestingUtility;"); htu_ = env_->CallStaticObjectMethod(testing_util_class_, mid); // this should be converted to a globalref I think to avoid the underlying java obj getting // GC'ed @@ -128,12 +98,12 @@ void MiniCluster::Setup() { exit(-1); } get_conn_mid_ = env_->GetMethodID(testing_util_class_, "getConnection", - "()Lorg/apache/hadoop/hbase/client/Connection;"); - jclass connClass = env_->FindClass("org/apache/hadoop/hbase/client/Connection"); + "()Lorg/apache/hadoop/hbase/client/Connection;"); + jclass conn_class = env_->FindClass("org/apache/hadoop/hbase/client/Connection"); get_admin_mid_ = - env_->GetMethodID(connClass, "getAdmin", "()Lorg/apache/hadoop/hbase/client/Admin;"); + env_->GetMethodID(conn_class, "getAdmin", "()Lorg/apache/hadoop/hbase/client/Admin;"); get_table_mid_ = env_->GetMethodID( - connClass, "getTable", + conn_class, "getTable", "(Lorg/apache/hadoop/hbase/TableName;)Lorg/apache/hadoop/hbase/client/Table;"); if (get_table_mid_ == NULL) { LOG(INFO) << "Couldn't find getConnection"; @@ -145,9 +115,10 @@ void MiniCluster::Setup() { LOG(INFO) << "Couldn't find move"; exit(-1); } - create_table_mid_ = env_->GetMethodID(testing_util_class_, "createTable", - "(Lorg/apache/hadoop/hbase/TableName;Ljava/lang/String;)Lorg/" - "apache/hadoop/hbase/client/Table;"); + create_table_mid_ = + env_->GetMethodID(testing_util_class_, "createTable", + "(Lorg/apache/hadoop/hbase/TableName;Ljava/lang/String;)Lorg/" + "apache/hadoop/hbase/client/Table;"); create_table_with_split_mid_ = env_->GetMethodID( testing_util_class_, "createTable", "(Lorg/apache/hadoop/hbase/TableName;[[B[[B)Lorg/apache/hadoop/hbase/client/Table;"); @@ -166,12 +137,13 @@ void MiniCluster::Setup() { jclass hbaseMiniClusterClass = env_->FindClass("org/apache/hadoop/hbase/MiniHBaseCluster"); stop_rs_mid_ = env_->GetMethodID(hbaseMiniClusterClass, "stopRegionServer", - "(I)Lorg/apache/hadoop/hbase/util/JVMClusterUtil$RegionServerThread;"); + "(I)Lorg/apache/hadoop/hbase/util/JVMClusterUtil$RegionServerThread;"); get_conf_mid_ = env_->GetMethodID(hbaseMiniClusterClass, "getConfiguration", - "()Lorg/apache/hadoop/conf/Configuration;"); + "()Lorg/apache/hadoop/conf/Configuration;"); conf_class_ = env_->FindClass("org/apache/hadoop/conf/Configuration"); - set_conf_mid_ = env_->GetMethodID(conf_class_, "set", "(Ljava/lang/String;Ljava/lang/String;)V"); + set_conf_mid_ = + env_->GetMethodID(conf_class_, "set", "(Ljava/lang/String;Ljava/lang/String;)V"); if (set_conf_mid_ == NULL) { LOG(INFO) << "Couldn't find method getConf in MiniHBaseCluster"; exit(-1); @@ -182,7 +154,7 @@ void MiniCluster::Setup() { put_mid_ = env_->GetMethodID(tableClass, "put", "(Lorg/apache/hadoop/hbase/client/Put;)V"); jclass connFactoryClass = env_->FindClass("org/apache/hadoop/hbase/client/ConnectionFactory"); create_conn_mid_ = env_->GetStaticMethodID(connFactoryClass, "createConnection", - "()Lorg/apache/hadoop/hbase/client/Connection;"); + "()Lorg/apache/hadoop/hbase/client/Connection;"); if (create_conn_mid_ == NULL) { LOG(INFO) << "Couldn't find createConnection"; exit(-1); @@ -209,49 +181,49 @@ JNIEnv *MiniCluster::env() { return env_; } // converts C char* to Java byte[] -jbyteArray MiniCluster::StrToByteChar(const std::string& str) { - char *p = const_cast(str.c_str()); - int n = 0; - while (*p++) { - n++; - } - if (n == NULL) return NULL; +jbyteArray MiniCluster::StrToByteChar(const std::string &str) { + char *p = const_cast(str.c_str()); + int n = str.length(); jbyteArray arr = env_->NewByteArray(n); env_->SetByteArrayRegion(arr, 0, n, reinterpret_cast(str.c_str())); return arr; } -jobject MiniCluster::CreateTable(std::string tblNam, std::string familyName) { - jstring tblNameStr = env_->NewStringUTF(tblNam.c_str()); - jobject tblName = env_->CallStaticObjectMethod(table_name_class_, tbl_name_value_of_mid_, tblNameStr); - jstring famStr = env_->NewStringUTF(familyName.c_str()); - jobject tbl = env_->CallObjectMethod(htu_, create_table_mid_, tblName, famStr); - return tbl; +jobject MiniCluster::CreateTable(const std::string &table, const std::string &family) { + jstring table_name_str = env_->NewStringUTF(table.c_str()); + jobject table_name = + env_->CallStaticObjectMethod(table_name_class_, tbl_name_value_of_mid_, table_name_str); + jstring family_str = env_->NewStringUTF(family.c_str()); + jobject table_obj = env_->CallObjectMethod(htu_, create_table_mid_, table_name, family_str); + return table_obj; } -jobject MiniCluster::CreateTable(std::string tblNam, std::string familyName, std::string key1, - std::string key2) { - jstring tblNameStr = env_->NewStringUTF(tblNam.c_str()); - jobject tblName = env_->CallStaticObjectMethod(table_name_class_, tbl_name_value_of_mid_, tblNameStr); - jclass arrayElemType = env_->FindClass("[B"); - jobjectArray famArray = env_->NewObjectArray(1, arrayElemType, env_->NewByteArray(1)); - env_->SetObjectArrayElement(famArray, 0, StrToByteChar(familyName)); +jobject MiniCluster::CreateTable(const std::string &table, const std::string &family, + const std::vector &keys) { + jstring table_name_str = env_->NewStringUTF(table.c_str()); + jobject table_name = + env_->CallStaticObjectMethod(table_name_class_, tbl_name_value_of_mid_, table_name_str); + jclass array_element_type = env_->FindClass("[B"); - int len = 2; - if (key2.empty()) len = 1; - jobjectArray keyArray = env_->NewObjectArray(len, arrayElemType, env_->NewByteArray(1)); + jobjectArray family_array = env_->NewObjectArray(1, array_element_type, env_->NewByteArray(1)); + env_->SetObjectArrayElement(family_array, 0, StrToByteChar(family)); - env_->SetObjectArrayElement(keyArray, 0, StrToByteChar(key1)); - if (!key2.empty()) { - env_->SetObjectArrayElement(keyArray, 1, StrToByteChar(key2)); + jobjectArray key_array = + env_->NewObjectArray(keys.size(), array_element_type, env_->NewByteArray(1)); + + int i = 0; + for (auto key : keys) { + env_->SetObjectArrayElement(key_array, i++, StrToByteChar(key)); } - jobject tbl = env_->CallObjectMethod(htu_, create_table_with_split_mid_, tblName, famArray, keyArray); + + jobject tbl = env_->CallObjectMethod(htu_, create_table_with_split_mid_, table_name, family_array, + key_array); return tbl; } -jobject MiniCluster::StopRegionServer(jobject cluster, int idx) { +jobject MiniCluster::StopRegionServer(int idx) { env(); - return env_->CallObjectMethod(cluster, stop_rs_mid_, (jint)idx); + return env_->CallObjectMethod(cluster_, stop_rs_mid_, (jint)idx); } // returns the Configuration for the cluster @@ -267,8 +239,9 @@ jobject MiniCluster::admin() { return admin; } -jobject MiniCluster::TablePut(const std::string table, const std::string row, const std::string fam, - const std::string col, const std::string value) { +jobject MiniCluster::TablePut(const std::string &table, const std::string &row, + const std::string &family, const std::string &column, + const std::string &value) { env(); jobject conn = env_->CallObjectMethod(htu(), get_conn_mid_); jobject put = env_->NewObject(put_class_, put_ctor_, StrToByteChar(row)); @@ -276,47 +249,30 @@ jobject MiniCluster::TablePut(const std::string table, const std::string row, co LOG(INFO) << "Couldn't create Put"; exit(-1); } - env_->CallObjectMethod(put, add_col_mid_, StrToByteChar(fam), StrToByteChar(col), - StrToByteChar(value)); - jobject tblName = - env_->CallStaticObjectMethod(table_name_class_, tbl_name_value_of_mid_, - env_->NewStringUTF(table.c_str())); - jobject tableObj = env_->CallObjectMethod(conn, get_table_mid_, tblName); - env_->CallObjectMethod(tableObj, put_mid_, put); - return tableObj; + env_->CallObjectMethod(put, add_col_mid_, StrToByteChar(family), StrToByteChar(column), + StrToByteChar(value)); + jobject table_name_obj = env_->CallStaticObjectMethod(table_name_class_, tbl_name_value_of_mid_, + env_->NewStringUTF(table.c_str())); + jobject table_obj = env_->CallObjectMethod(conn, get_table_mid_, table_name_obj); + env_->CallObjectMethod(table_obj, put_mid_, put); + return table_obj; } // moves region to server -void MiniCluster::MoveRegion(std::string region, std::string server) { +void MiniCluster::MoveRegion(const std::string ®ion, const std::string &server) { jobject admin_ = admin(); - env_->CallObjectMethod(admin_, move_mid_, StrToByteChar(region), - StrToByteChar(server)); + env_->CallObjectMethod(admin_, move_mid_, StrToByteChar(region), StrToByteChar(server)); } -jobject MiniCluster::StartCluster(int numRegionServers, std::string conf_path) { +jobject MiniCluster::StartCluster(int num_region_servers) { env(); jmethodID mid = env_->GetMethodID(testing_util_class_, "startMiniCluster", - "(I)Lorg/apache/hadoop/hbase/MiniHBaseCluster;"); + "(I)Lorg/apache/hadoop/hbase/MiniHBaseCluster;"); if (mid == NULL) { LOG(INFO) << "Couldn't find method startMiniCluster in the class HBaseTestingUtility"; exit(-1); } - cluster_ = env_->CallObjectMethod(htu(), mid, (jint)numRegionServers); - jobject conf = GetConf(); - jstring jport = (jstring)env_->CallObjectMethod( - conf, conf_get_mid_, env_->NewStringUTF("hbase.zookeeper.property.clientPort")); - const char *port = env_->GetStringUTFChars(jport, 0); - LOG(INFO) << "retrieved port " << port; - std::string quorum("localhost:"); - env_->CallObjectMethod(conf, set_conf_mid_, env_->NewStringUTF("hbase.zookeeper.quorum"), - env_->NewStringUTF((quorum + port).c_str())); - if (!conf_path.empty()) { - // Directory will be created if not present - if (!boost::filesystem::exists(conf_path)) { - boost::filesystem::create_directories(conf_path); - } - WriteConf(conf, conf_path + "/hbase-site.xml"); - } + cluster_ = env_->CallObjectMethod(htu(), mid, static_cast(num_region_servers)); return cluster_; } @@ -324,16 +280,16 @@ void MiniCluster::StopCluster() { env(); jmethodID mid = env_->GetMethodID(testing_util_class_, "shutdownMiniCluster", "()V"); env_->CallVoidMethod(htu(), mid); - if (jvm != NULL) { - jvm->DestroyJavaVM(); - jvm = NULL; + if (jvm_ != NULL) { + jvm_->DestroyJavaVM(); + jvm_ = NULL; } } -const std::string MiniCluster::GetConfValue(std::string key) { +const std::string MiniCluster::GetConfValue(const std::string &key) { jobject conf = GetConf(); - jstring jval = (jstring)env_->CallObjectMethod(conf, conf_get_mid_, - env_->NewStringUTF(key.c_str())); + jstring jval = + (jstring)env_->CallObjectMethod(conf, conf_get_mid_, env_->NewStringUTF(key.c_str())); const char *val = env_->GetStringUTFChars(jval, 0); return val; } diff --git hbase-native-client/test-util/mini-cluster.h hbase-native-client/test-util/mini-cluster.h index 54842e1..a9502b5 100644 --- hbase-native-client/test-util/mini-cluster.h +++ hbase-native-client/test-util/mini-cluster.h @@ -19,27 +19,29 @@ #pragma once #include +#include #include "jni.h" namespace hbase { class MiniCluster { public: - jobject StartCluster(int numRegionServers, std::string conf_path); + jobject StartCluster(int32_t num_region_servers); void StopCluster(); - jobject CreateTable(std::string tblNam, std::string familyName); - jobject CreateTable(std::string tblNam, std::string familyName, std::string key1, std::string k2); - jobject StopRegionServer(jobject cluster, int idx); + jobject CreateTable(const std::string &table, const std::string &family); + jobject CreateTable(const std::string &table, const std::string &family, + const std::vector &keys); + jobject StopRegionServer(int idx); // moves region to server - void MoveRegion(std::string region, std::string server); + void MoveRegion(const std::string ®ion, const std::string &server); // returns the Configuration instance for the cluster jobject GetConf(); // returns the value for config key retrieved from cluster - const std::string GetConfValue(std::string key); + const std::string GetConfValue(const std::string &key); // Does Put into table for family fam, qualifier col with value - jobject TablePut(const std::string table, const std::string row, const std::string fam, - const std::string col, const std::string value); + jobject TablePut(const std::string &table, const std::string &row, const std::string &family, + const std::string &column, const std::string &value); private: JNIEnv *env_; @@ -66,13 +68,12 @@ class MiniCluster { jobject htu_; jobject cluster_; pthread_mutex_t count_mutex_; - JavaVM *jvm; + JavaVM *jvm_; JNIEnv *CreateVM(JavaVM **jvm); - void WriteConf(jobject conf, const std::string& filepath); void Setup(); jobject htu(); JNIEnv *env(); - jbyteArray StrToByteChar(const std::string& str); + jbyteArray StrToByteChar(const std::string &str); jobject admin(); }; } /*namespace hbase*/ diff --git hbase-native-client/test-util/test-util.cc hbase-native-client/test-util/test-util.cc index 13c4ec4..dfe4260 100644 --- hbase-native-client/test-util/test-util.cc +++ hbase-native-client/test-util/test-util.cc @@ -22,6 +22,8 @@ #include +#include "core/zk-util.h" + using hbase::TestUtil; using folly::Random; @@ -42,33 +44,28 @@ std::string TestUtil::RandString(int len) { return s; } -TestUtil::TestUtil() : TestUtil::TestUtil(2, "") {} - -TestUtil::TestUtil(int servers, const std::string& confPath) - : temp_dir_(TestUtil::RandString()), numRegionServers(servers), conf_path(confPath) { - auto p = temp_dir_.path().string(); - StartMiniCluster(2); - std::string quorum("localhost:"); - const std::string port = mini->GetConfValue("hbase.zookeeper.property.clientPort"); - conf()->Set("hbase.zookeeper.quorum", quorum + port); -} +TestUtil::TestUtil() : temp_dir_(TestUtil::RandString()) {} TestUtil::~TestUtil() { StopMiniCluster(); } -void TestUtil::StartMiniCluster(int num_region_servers) { - mini = std::make_unique(); - mini->StartCluster(num_region_servers, conf_path); +void TestUtil::StartMiniCluster(int32_t num_region_servers) { + mini_ = std::make_unique(); + mini_->StartCluster(num_region_servers); + + conf()->Set(ZKUtil::kHBaseZookeeperQuorum_, mini_->GetConfValue(ZKUtil::kHBaseZookeeperQuorum_)); + conf()->Set(ZKUtil::kHBaseZookeeperClientPort_, + mini_->GetConfValue(ZKUtil::kHBaseZookeeperClientPort_)); } -void TestUtil::StopMiniCluster() { mini->StopCluster(); } +void TestUtil::StopMiniCluster() { mini_->StopCluster(); } -void TestUtil::CreateTable(std::string tblNam, std::string familyName) { - mini->CreateTable(tblNam, familyName); +void TestUtil::CreateTable(const std::string &table, const std::string &family) { + mini_->CreateTable(table, family); } -void TestUtil::CreateTable(std::string tblNam, std::string familyName, std::string key1, - std::string k2) { - mini->CreateTable(tblNam, familyName, key1, k2); +void TestUtil::CreateTable(const std::string &table, const std::string &family, + const std::vector &keys) { + mini_->CreateTable(table, family, keys); } -void TestUtil::TablePut(std::string table, std::string row, std::string fam, std::string col, - std::string value) { - mini->TablePut(table, row, fam, col, value); +void TestUtil::TablePut(const std::string &table, const std::string &row, const std::string &family, + const std::string &column, const std::string &value) { + mini_->TablePut(table, row, family, column, value); } diff --git hbase-native-client/test-util/test-util.h hbase-native-client/test-util/test-util.h index 1734870..611404d 100644 --- hbase-native-client/test-util/test-util.h +++ hbase-native-client/test-util/test-util.h @@ -24,6 +24,7 @@ #include #include #include +#include #include "core/configuration.h" #include "test-util/mini-cluster.h" @@ -33,14 +34,7 @@ namespace hbase { */ class TestUtil { public: - /** - * Creating a TestUtil will spin up a cluster. - */ TestUtil(); - /** - * Creating a TestUtil will spin up a cluster with numRegionServers region servers. - */ - TestUtil(int numRegionServers, const std::string& confPath); /** * Destroying a TestUtil will spin down a cluster and remove the test dir. @@ -61,19 +55,18 @@ class TestUtil { /** * Starts mini hbase cluster with specified number of region servers */ - void StartMiniCluster(int num_region_servers); + void StartMiniCluster(int32_t num_region_servers); void StopMiniCluster(); - void CreateTable(std::string tblNam, std::string familyName); - void CreateTable(std::string tblNam, std::string familyName, std::string key1, std::string k2); - void TablePut(std::string table, std::string row, std::string fam, std::string col, - std::string value); + void CreateTable(const std::string &table, const std::string &family); + void CreateTable(const std::string &table, const std::string &family, + const std::vector &keys); + void TablePut(const std::string &table, const std::string &row, const std::string &family, + const std::string &column, const std::string &value); private: - std::unique_ptr mini; + std::unique_ptr mini_; folly::test::TemporaryDirectory temp_dir_; - int numRegionServers = 2; - std::string conf_path; std::shared_ptr conf_ = std::make_shared(); }; } // namespace hbase diff --git hbase-native-client/third-party/BUCK hbase-native-client/third-party/BUCK index a55a6fb..f37eb4e 100644 --- hbase-native-client/third-party/BUCK +++ hbase-native-client/third-party/BUCK @@ -112,6 +112,4 @@ cxx_library( ('googletest/googlemock', 'src/*.cc'), ]), exported_deps=dynamic_rules, - visibility=[ - 'PUBLIC', - ],) + visibility=['PUBLIC',],)