diff --git a/hbase-native-client/Makefile b/hbase-native-client/Makefile index 84ae556..6930ff9 100644 --- a/hbase-native-client/Makefile +++ b/hbase-native-client/Makefile @@ -26,13 +26,13 @@ MODULES = connection core serde test-util utils security SRC_DIR = $(MODULES) DEBUG_BUILD_DIR = $(addprefix $(DEBUG_PATH)/,$(MODULES)) RELEASE_BUILD_DIR = $(addprefix $(RELEASE_PATH)/,$(MODULES)) -INCLUDE_DIR = . build/ +INCLUDE_DIR = . build/ $(JAVA_HOME)/include/ $(JAVA_HOME)/include/linux #flags to pass to the CPP compiler & linker CPPFLAGS_DEBUG = -D_GLIBCXX_USE_CXX11_ABI=0 -g -Wall -std=c++14 -pedantic -fPIC CPPFLAGS_RELEASE = -D_GLIBCXX_USE_CXX11_ABI=0 -DNDEBUG -O2 -Wall -std=c++14 -pedantic -fPIC LDFLAGS = -lprotobuf -lzookeeper_mt -lsasl2 -lfolly -lwangle -LINKFLAG = -shared +LINKFLAG = -shared -L$(JAVA_HOME)/jre/lib/amd64/server -ljvm #define list of source files and object files ALLSRC = $(foreach sdir,$(SRC_DIR),$(wildcard $(sdir)/*.cc)) diff --git a/hbase-native-client/connection/BUCK b/hbase-native-client/connection/BUCK index 19536d5..bc05be0 100644 --- a/hbase-native-client/connection/BUCK +++ b/hbase-native-client/connection/BUCK @@ -50,8 +50,14 @@ cxx_library( "//third-party:wangle", ], compiler_flags=['-Weffc++'], - visibility=['//core/...',],) + visibility=[ + '//core/...', + ],) cxx_test( name="connection-pool-test", - srcs=["connection-pool-test.cc",], - deps=[":connection",],) + srcs=[ + "connection-pool-test.cc", + ], + deps=[ + ":connection", + ],) diff --git a/hbase-native-client/connection/connection-factory.cc b/hbase-native-client/connection/connection-factory.cc index 2f7e75c..832b00f 100644 --- a/hbase-native-client/connection/connection-factory.cc +++ b/hbase-native-client/connection/connection-factory.cc @@ -31,11 +31,10 @@ using std::chrono::milliseconds; using std::chrono::nanoseconds; ConnectionFactory::ConnectionFactory(std::shared_ptr io_pool, - std::shared_ptr codec, - nanoseconds connect_timeout) + std::shared_ptr codec, nanoseconds connect_timeout) : connect_timeout_(connect_timeout), - io_pool_(io_pool), - pipeline_factory_(std::make_shared(codec)) {} + io_pool_(io_pool), + pipeline_factory_(std::make_shared(codec)) {} std::shared_ptr> ConnectionFactory::MakeBootstrap() { auto client = std::make_shared>(); diff --git a/hbase-native-client/connection/connection-factory.h b/hbase-native-client/connection/connection-factory.h index fbcb6ef..32d0bf7 100644 --- a/hbase-native-client/connection/connection-factory.h +++ b/hbase-native-client/connection/connection-factory.h @@ -44,8 +44,7 @@ class ConnectionFactory { * There should only be one ConnectionFactory per client. */ ConnectionFactory(std::shared_ptr io_pool, - std::shared_ptr codec, - nanoseconds connect_timeout = nanoseconds(0)); + std::shared_ptr codec, nanoseconds connect_timeout = nanoseconds(0)); /** Default Destructor */ virtual ~ConnectionFactory() = default; diff --git a/hbase-native-client/connection/connection-pool.cc b/hbase-native-client/connection/connection-pool.cc index b18ee89..b712a28 100644 --- a/hbase-native-client/connection/connection-pool.cc +++ b/hbase-native-client/connection/connection-pool.cc @@ -34,8 +34,7 @@ using folly::SharedMutexWritePriority; using folly::SocketAddress; ConnectionPool::ConnectionPool(std::shared_ptr io_executor, - std::shared_ptr codec, - nanoseconds connect_timeout) + std::shared_ptr codec, nanoseconds connect_timeout) : cf_(std::make_shared(io_executor, codec, connect_timeout)), clients_(), connections_(), diff --git a/hbase-native-client/connection/rpc-client.h b/hbase-native-client/connection/rpc-client.h index 5c11ab5..3ebc21a 100644 --- a/hbase-native-client/connection/rpc-client.h +++ b/hbase-native-client/connection/rpc-client.h @@ -54,8 +54,8 @@ class RpcClient : public std::enable_shared_from_this { friend class RpcChannelImplementation; public: - RpcClient(std::shared_ptr io_executor, - std::shared_ptr codec, nanoseconds connect_timeout); + RpcClient(std::shared_ptr io_executor, std::shared_ptr codec, + nanoseconds connect_timeout); virtual ~RpcClient() { Close(); } diff --git a/hbase-native-client/core/BUCK b/hbase-native-client/core/BUCK index e541d8f..8998fe2 100644 --- a/hbase-native-client/core/BUCK +++ b/hbase-native-client/core/BUCK @@ -65,11 +65,15 @@ cxx_library( "//third-party:wangle", "//third-party:zookeeper_mt", ], - compiler_flags=['-Weffc++'], - visibility=['PUBLIC',],) + compiler_flags=['-Weffc++', '-ggdb'], + visibility=[ + 'PUBLIC', + ],) cxx_test( name="location-cache-test", - srcs=["location-cache-test.cc",], + srcs=[ + "location-cache-test.cc", + ], deps=[ ":core", "//test-util:test-util", @@ -77,12 +81,18 @@ cxx_test( run_test_separately=True,) cxx_test( name="cell-test", - srcs=["cell-test.cc",], - deps=[":core",], + srcs=[ + "cell-test.cc", + ], + deps=[ + ":core", + ], run_test_separately=True,) cxx_test( name="filter-test", - srcs=["filter-test.cc",], + srcs=[ + "filter-test.cc", + ], deps=[ ":core", "//if:if", @@ -92,37 +102,63 @@ cxx_test( run_test_separately=True,) cxx_test( name="get-test", - srcs=["get-test.cc",], - deps=[":core",], + srcs=[ + "get-test.cc", + ], + deps=[ + ":core", + ], run_test_separately=True,) cxx_test( name="time_range-test", - srcs=["time_range-test.cc",], - deps=[":core",], + srcs=[ + "time_range-test.cc", + ], + deps=[ + ":core", + ], run_test_separately=True,) cxx_test( name="configuration-test", - srcs=["configuration-test.cc",], - deps=[":core",], + srcs=[ + "configuration-test.cc", + ], + deps=[ + ":core", + ], run_test_separately=True,) cxx_test( name="hbase_configuration-test", - srcs=["hbase_configuration-test.cc",], - deps=[":core",], + srcs=[ + "hbase_configuration-test.cc", + ], + deps=[ + ":core", + ], run_test_separately=True,) cxx_test( name="scan-test", - srcs=["scan-test.cc",], - deps=[":core",], + srcs=[ + "scan-test.cc", + ], + deps=[ + ":core", + ], run_test_separately=True,) cxx_test( name="result-test", - srcs=["result-test.cc",], - deps=[":core",], + srcs=[ + "result-test.cc", + ], + deps=[ + ":core", + ], run_test_separately=True,) cxx_test( name="request_converter-test", - srcs=["request_converter-test.cc",], + srcs=[ + "request_converter-test.cc", + ], deps=[ ":core", "//connection:connection", @@ -131,7 +167,9 @@ cxx_test( run_test_separately=True,) cxx_test( name="client-test", - srcs=["client-test.cc",], + srcs=[ + "client-test.cc", + ], deps=[ ":core", "//if:if", @@ -141,5 +179,7 @@ cxx_test( run_test_separately=True,) cxx_binary( name="simple-client", - srcs=["simple-client.cc",], + srcs=[ + "simple-client.cc", + ], deps=[":core", "//connection:connection"],) diff --git a/hbase-native-client/core/client-test.cc b/hbase-native-client/core/client-test.cc index 0a45fff..496cd70 100644 --- a/hbase-native-client/core/client-test.cc +++ b/hbase-native-client/core/client-test.cc @@ -27,7 +27,7 @@ #include "serde/table-name.h" #include "test-util/test-util.h" -class ClientTest { +class ClientTest : public ::testing::Test { public: const static std::string kDefHBaseConfPath; @@ -45,10 +45,6 @@ class ClientTest { static void CreateHBaseConf(const std::string &dir, const std::string &file, const std::string xml_data) { - // Directory will be created if not present - if (!boost::filesystem::exists(dir)) { - boost::filesystem::create_directories(dir); - } // Remove temp file always boost::filesystem::remove((dir + file).c_str()); WriteDataToFile((dir + file), xml_data); @@ -57,10 +53,16 @@ class ClientTest { static void CreateHBaseConfWithEnv() { // Creating Empty Config Files so that we dont get a Configuration exception @Client CreateHBaseConf(kDefHBaseConfPath, kHBaseDefaultXml, kHBaseXmlData); - CreateHBaseConf(kDefHBaseConfPath, kHBaseSiteXml, kHBaseXmlData); + // the hbase-site.xml would be persisted by MiniCluster setenv("HBASE_CONF", kDefHBaseConfPath.c_str(), 1); } + static std::unique_ptr test_util; + + static void SetUpTestCase() { + test_util = std::make_unique(2, ClientTest::kDefHBaseConfPath.c_str()); + } }; +std::unique_ptr ClientTest::test_util = nullptr; const std::string ClientTest::kDefHBaseConfPath("./build/test-data/client-test/conf/"); @@ -109,16 +111,15 @@ TEST(Client, DefaultConfiguration) { client.Close(); } -TEST(Client, Get) { +TEST_F(ClientTest, Get) { // Remove already configured env if present. unsetenv("HBASE_CONF"); ClientTest::CreateHBaseConfWithEnv(); // Using TestUtil to populate test data - hbase::TestUtil *test_util = new hbase::TestUtil(); - test_util->RunShellCmd( - "create 't', 'd'; put 't', 'test2', 'd:2', 'value2'; put 't', 'test2', 'd:extra', 'value for " - "extra'"); + ClientTest::test_util->CreateTable("t", "d"); + ClientTest::test_util->TablePut("t", "test2", "d", "2", "value2"); + ClientTest::test_util->TablePut("t", "test2", "d", "extra", "value for extra"); // Create TableName and Row to be fetched from HBase auto tn = folly::to("t"); @@ -145,7 +146,6 @@ TEST(Client, Get) { // The connection stays open and we don't want that. // So we are stopping the connection. // We can remove this once we have fixed the folly part - delete test_util; // Test the values, should be same as in put executed on hbase shell ASSERT_TRUE(!result->IsEmpty()) << "Result shouldn't be empty."; @@ -163,7 +163,7 @@ TEST(Client, GetForNonExistentTable) { ClientTest::CreateHBaseConfWithEnv(); // Using TestUtil to populate test data - hbase::TestUtil *test_util = new hbase::TestUtil(); + // hbase::TestUtil *test_util = new hbase::TestUtil(2, ClientTest::kDefHBaseConfPath.c_str()); // Create TableName and Row to be fetched from HBase auto tn = folly::to("t_not_exists"); @@ -190,20 +190,20 @@ TEST(Client, GetForNonExistentTable) { // The connection stays open and we don't want that. // So we are stopping the connection. // We can remove this once we have fixed the folly part - delete test_util; + // delete test_util; table->Close(); client.Close(); } -TEST(Client, GetForNonExistentRow) { +TEST_F(ClientTest, GetForNonExistentRow) { // Remove already configured env if present. unsetenv("HBASE_CONF"); ClientTest::CreateHBaseConfWithEnv(); // Using TestUtil to populate test data - hbase::TestUtil *test_util = new hbase::TestUtil(); - test_util->RunShellCmd("create 't_exists', 'd'"); + // hbase::TestUtil *test_util = new hbase::TestUtil(2, ClientTest::kDefHBaseConfPath.c_str()); + ClientTest::test_util->CreateTable("t_exists", "d"); // Create TableName and Row to be fetched from HBase auto tn = folly::to("t_exists"); @@ -231,7 +231,6 @@ TEST(Client, GetForNonExistentRow) { // The connection stays open and we don't want that. // So we are stopping the connection. // We can remove this once we have fixed the folly part - delete test_util; table->Close(); client.Close(); diff --git a/hbase-native-client/core/client.cc b/hbase-native-client/core/client.cc index 240da72..f0483ef 100644 --- a/hbase-native-client/core/client.cc +++ b/hbase-native-client/core/client.cc @@ -57,7 +57,8 @@ void Client::init(const hbase::Configuration &conf) { } else { LOG(WARNING) << "Not using RPC Cell Codec"; } - rpc_client_ = std::make_shared(io_executor_, codec, conn_conf_->connect_timeout()); + rpc_client_ = + std::make_shared(io_executor_, codec, conn_conf_->connect_timeout()); location_cache_ = std::make_shared(conf_, cpu_executor_, rpc_client_->connection_pool()); } diff --git a/hbase-native-client/core/client.h b/hbase-native-client/core/client.h index a96d6f3..e73ab70 100644 --- a/hbase-native-client/core/client.h +++ b/hbase-native-client/core/client.h @@ -89,7 +89,7 @@ class Client { bool is_closed_ = false; /** Methods */ - void init(const hbase::Configuration &conf); + void init(const hbase::Configuration& conf); }; } // namespace hbase diff --git a/hbase-native-client/core/filter-test.cc b/hbase-native-client/core/filter-test.cc index ff683b6..6529b5a 100644 --- a/hbase-native-client/core/filter-test.cc +++ b/hbase-native-client/core/filter-test.cc @@ -39,7 +39,9 @@ using hbase::Comparator; class FilterTest : public ::testing::Test { protected: - static void SetUpTestCase() { test_util_ = std::make_unique(); } + static void SetUpTestCase() { + test_util_ = std::make_unique(); + } static void TearDownTestCase() { test_util_.release(); } @@ -49,13 +51,15 @@ class FilterTest : public ::testing::Test { static std::unique_ptr test_util_; }; + std::unique_ptr FilterTest::test_util_ = nullptr; TEST_F(FilterTest, GetWithColumnPrefixFilter) { // write row1 with 3 columns (column_1, column_2, and foo_column) - FilterTest::test_util_->RunShellCmd( - "create 't', 'd'; put 't', 'row1', 'd:column_1', 'value1'; put 't', 'row1', 'd:column_2', " - "'value2'; put 't', 'row1', 'd:foo_column', 'value3'"); + FilterTest::test_util_->CreateTable("t", "d"); + FilterTest::test_util_->TablePut("t", "row1", "d", "column_1", "value1"); + FilterTest::test_util_->TablePut("t", "row1", "d", "column_2", "value2"); + FilterTest::test_util_->TablePut("t", "row1", "d", "foo_column", "value3"); // Create TableName and Row to be fetched from HBase auto tn = folly::to("t"); @@ -70,7 +74,7 @@ TEST_F(FilterTest, GetWithColumnPrefixFilter) { get_two.SetFilter(FilterFactory::ColumnPrefixFilter("column_")); // Create a client - hbase::Client client(Configuration{}); + hbase::Client client(*(FilterTest::test_util_->conf())); // Get connection to HBase Table auto table = client.Table(tn); @@ -101,9 +105,10 @@ TEST_F(FilterTest, GetWithColumnPrefixFilter) { TEST_F(FilterTest, GetWithQualifierFilter) { // write row1 with 3 columns (a,b,c) - FilterTest::test_util_->RunShellCmd( - "create 't1', 'd'; put 't1', 'row1', 'd:a', 'value1'; put 't1', 'row1', 'd:b', " - "'value2'; put 't1', 'row1', 'd:c', 'value3'"); + FilterTest::test_util_->CreateTable("t1", "d"); + FilterTest::test_util_->TablePut("t1", "row1", "d", "a", "value1"); + FilterTest::test_util_->TablePut("t1", "row1", "d", "b", "value2"); + FilterTest::test_util_->TablePut("t1", "row1", "d", "c", "value3"); // Create TableName and Row to be fetched from HBase auto tn = folly::to("t1"); @@ -115,7 +120,7 @@ TEST_F(FilterTest, GetWithQualifierFilter) { *ComparatorFactory::BinaryComparator("b"))); // Create a client - hbase::Client client(Configuration{}); + hbase::Client client(*(FilterTest::test_util_->conf())); // Get connection to HBase Table auto table = client.Table(tn); diff --git a/hbase-native-client/core/filter.h b/hbase-native-client/core/filter.h index b5b7133..10accaa 100644 --- a/hbase-native-client/core/filter.h +++ b/hbase-native-client/core/filter.h @@ -20,9 +20,9 @@ #pragma once #include +#include #include #include -#include #include #include "if/Comparator.pb.h" diff --git a/hbase-native-client/core/location-cache-test.cc b/hbase-native-client/core/location-cache-test.cc index 9a778c6..75a56d1 100644 --- a/hbase-native-client/core/location-cache-test.cc +++ b/hbase-native-client/core/location-cache-test.cc @@ -30,13 +30,29 @@ using namespace hbase; using namespace std::chrono; -TEST(LocationCacheTest, TestGetMetaNodeContents) { - TestUtil test_util{}; +class LocationCacheTest : public ::testing::Test { + protected: + static void SetUpTestCase() { + test_util_ = std::make_unique(); + } + static void TearDownTestCase() { test_util_.release(); } + + virtual void SetUp() {} + virtual void TearDown() {} + + public: + static std::unique_ptr test_util_; +}; + +std::unique_ptr LocationCacheTest::test_util_ = nullptr; + + +TEST_F(LocationCacheTest, TestGetMetaNodeContents) { auto cpu = std::make_shared(4); auto io = std::make_shared(4); auto codec = std::make_shared(); auto cp = std::make_shared(io, codec); - LocationCache cache{test_util.conf(), cpu, cp}; + LocationCache cache{LocationCacheTest::test_util_->conf(), cpu, cp}; auto f = cache.LocateMeta(); auto result = f.get(); ASSERT_FALSE(f.hasException()); @@ -47,19 +63,18 @@ TEST(LocationCacheTest, TestGetMetaNodeContents) { cp->Close(); } -TEST(LocationCacheTest, TestGetRegionLocation) { - TestUtil test_util{}; +TEST_F(LocationCacheTest, TestGetRegionLocation) { auto cpu = std::make_shared(4); auto io = std::make_shared(4); auto codec = std::make_shared(); auto cp = std::make_shared(io, codec); - LocationCache cache{test_util.conf(), cpu, cp}; + LocationCache cache{LocationCacheTest::test_util_->conf(), cpu, cp}; // If there is no table this should throw an exception auto tn = folly::to("t"); auto row = "test"; ASSERT_ANY_THROW(cache.LocateFromMeta(tn, row).get(milliseconds(1000))); - test_util.RunShellCmd("create 't', 'd'"); + LocationCacheTest::test_util_->CreateTable("t", "d"); auto loc = cache.LocateFromMeta(tn, row).get(milliseconds(1000)); ASSERT_TRUE(loc != nullptr); cpu->stop(); @@ -67,13 +82,12 @@ TEST(LocationCacheTest, TestGetRegionLocation) { cp->Close(); } -TEST(LocationCacheTest, TestCaching) { - TestUtil test_util{}; +TEST_F(LocationCacheTest, TestCaching) { auto cpu = std::make_shared(4); auto io = std::make_shared(4); auto codec = std::make_shared(); auto cp = std::make_shared(io, codec); - LocationCache cache{test_util.conf(), cpu, cp}; + LocationCache cache{LocationCacheTest::test_util_->conf(), cpu, cp}; auto tn_1 = folly::to("t1"); auto tn_2 = folly::to("t2"); @@ -83,8 +97,7 @@ TEST(LocationCacheTest, TestCaching) { // test location pulled from meta gets cached ASSERT_ANY_THROW(cache.LocateRegion(tn_1, row_a).get(milliseconds(1000))); ASSERT_ANY_THROW(cache.LocateFromMeta(tn_1, row_a).get(milliseconds(1000))); - - test_util.RunShellCmd("create 't1', 'd'"); + LocationCacheTest::test_util_->CreateTable("t1", "d"); ASSERT_FALSE(cache.IsLocationCached(tn_1, row_a)); auto loc = cache.LocateRegion(tn_1, row_a).get(milliseconds(1000)); @@ -92,7 +105,8 @@ TEST(LocationCacheTest, TestCaching) { ASSERT_EQ(loc, cache.GetCachedLocation(tn_1, row_a)); // test with two regions - test_util.RunShellCmd("create 't2', 'd', SPLITS => ['b']"); + std::string empty; + LocationCacheTest::test_util_->CreateTable("t2", "d", "b", empty); ASSERT_FALSE(cache.IsLocationCached(tn_2, "a")); loc = cache.LocateRegion(tn_2, "a").get(milliseconds(1000)); @@ -107,7 +121,7 @@ TEST(LocationCacheTest, TestCaching) { ASSERT_EQ(loc, cache.GetCachedLocation(tn_2, "ba")); // test with three regions - test_util.RunShellCmd("create 't3', 'd', SPLITS => ['b', 'c']"); + LocationCacheTest::test_util_->CreateTable("t3", "d", "b", "c"); ASSERT_FALSE(cache.IsLocationCached(tn_3, "c")); ASSERT_FALSE(cache.IsLocationCached(tn_3, "ca")); diff --git a/hbase-native-client/core/location-cache.cc b/hbase-native-client/core/location-cache.cc index da9f64a..21e07cc 100644 --- a/hbase-native-client/core/location-cache.cc +++ b/hbase-native-client/core/location-cache.cc @@ -106,7 +106,8 @@ ServerName LocationCache::ReadMetaLocation() { reinterpret_cast(buf->writableData()), &len, nullptr); if (zk_result != ZOK || len < 9) { LOG(ERROR) << "Error getting meta location."; - throw runtime_error("Error getting meta location"); + throw runtime_error("Error getting meta location. Quorum " + + conf_->Get(kHBaseZookeeperQuorum_, "")); } buf->append(len); diff --git a/hbase-native-client/security/BUCK b/hbase-native-client/security/BUCK index e176c90..d602ff3 100644 --- a/hbase-native-client/security/BUCK +++ b/hbase-native-client/security/BUCK @@ -19,7 +19,9 @@ # to a single server. cxx_library( name="security", - exported_headers=["user.h",], + exported_headers=[ + "user.h", + ], srcs=[], deps=[], compiler_flags=['-Weffc++'], diff --git a/hbase-native-client/serde/BUCK b/hbase-native-client/serde/BUCK index 38e7b4d..76d1c72 100644 --- a/hbase-native-client/serde/BUCK +++ b/hbase-native-client/serde/BUCK @@ -41,28 +41,54 @@ cxx_library( ":region-info-deserializer-test", ], compiler_flags=['-Weffc++'], - visibility=['PUBLIC',],) + visibility=[ + 'PUBLIC', + ],) cxx_test( name="table-name-test", - srcs=["table-name-test.cc",], - deps=[":serde",],) + srcs=[ + "table-name-test.cc", + ], + deps=[ + ":serde", + ],) cxx_test( name="server-name-test", - srcs=["server-name-test.cc",], - deps=[":serde",],) + srcs=[ + "server-name-test.cc", + ], + deps=[ + ":serde", + ],) cxx_test( name="client-serializer-test", - srcs=["client-serializer-test.cc",], - deps=[":serde",],) + srcs=[ + "client-serializer-test.cc", + ], + deps=[ + ":serde", + ],) cxx_test( name="client-deserializer-test", - srcs=["client-deserializer-test.cc",], - deps=[":serde",],) + srcs=[ + "client-deserializer-test.cc", + ], + deps=[ + ":serde", + ],) cxx_test( name="zk-deserializer-test", - srcs=["zk-deserializer-test.cc",], - deps=[":serde",],) + srcs=[ + "zk-deserializer-test.cc", + ], + deps=[ + ":serde", + ],) cxx_test( name="region-info-deserializer-test", - srcs=["region-info-deserializer-test.cc",], - deps=[":serde",],) + srcs=[ + "region-info-deserializer-test.cc", + ], + deps=[ + ":serde", + ],) diff --git a/hbase-native-client/test-util/BUCK b/hbase-native-client/test-util/BUCK index c6e41f1..4998614 100644 --- a/hbase-native-client/test-util/BUCK +++ b/hbase-native-client/test-util/BUCK @@ -14,16 +14,34 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import os cxx_library(name="test-util", exported_headers=[ - "test-util.h", + "test-util.h","mini-cluster.h" ], - srcs=["test-util.cc"], + srcs=["test-util.cc","mini-cluster.cc"], deps=[ "//third-party:folly", - "//core:core" + "//core:core", ], + preprocessor_flags= [ + '-I' + os.environ['JAVA_HOME'] + '/include', + '-I' + os.environ['JAVA_HOME'] + '/include/darwin', + '-I' + os.environ['JAVA_HOME'] + '/include/linux'], + exported_preprocessor_flags= [ + '-I' + os.environ['JAVA_HOME'] + '/include', + '-I' + os.environ['JAVA_HOME'] + '/include/darwin', + '-I' + os.environ['JAVA_HOME'] + '/include/linux'], + compiler_flags = [ + '-I' + os.environ['JAVA_HOME'] + '/include', + '-I' + os.environ['JAVA_HOME'] + '/include/darwin', + '-I' + os.environ['JAVA_HOME'] + '/include/linux', '-ggdb'], + linker_flags = ['-ljvm', + '-L' + os.environ['JAVA_HOME'] + '/jre/lib/amd64/server', '-ggdb'], + exported_linker_flags = ['-ljvm', + '-L' + os.environ['JAVA_HOME'] + '/jre/lib/amd64/server', + '-Wl,-rpath=' + os.environ['JAVA_HOME'] + '/jre/lib/amd64/server'], visibility=[ 'PUBLIC', ], diff --git a/hbase-native-client/test-util/test-util.cc b/hbase-native-client/test-util/test-util.cc index e355bdf..db19310 100644 --- a/hbase-native-client/test-util/test-util.cc +++ b/hbase-native-client/test-util/test-util.cc @@ -18,6 +18,7 @@ */ #include "test-util/test-util.h" +#include #include @@ -41,20 +42,39 @@ std::string TestUtil::RandString(int len) { return s; } -TestUtil::TestUtil() : temp_dir_(TestUtil::RandString()) { +TestUtil::TestUtil() : TestUtil::TestUtil(2, "") {} + +TestUtil::TestUtil(int servers, const char *confPath) + : temp_dir_(TestUtil::RandString()), numRegionServers(servers), conf_path(confPath) { auto p = temp_dir_.path().string(); - auto cmd = std::string{"bin/start-local-hbase.sh " + p}; - auto res_code = std::system(cmd.c_str()); - CHECK_EQ(res_code, 0); + jobject cluster = StartMiniCluster(2); + std::string quorum("localhost:"); + const std::string port = mini->GetConfValue("hbase.zookeeper.property.clientPort"); + conf()->Set("hbase.zookeeper.quorum", quorum + port); } -TestUtil::~TestUtil() { - auto res_code = std::system("bin/stop-local-hbase.sh"); - CHECK_EQ(res_code, 0); -} +TestUtil::~TestUtil() { StopMiniCluster(); } void TestUtil::RunShellCmd(const std::string &command) { auto cmd_string = folly::sformat("echo \"{}\" | ../bin/hbase shell", command); auto res_code = std::system(cmd_string.c_str()); CHECK_EQ(res_code, 0); } + +jobject TestUtil::StartMiniCluster(int num_region_servers) { + mini = std::make_unique(); + return mini->StartCluster(num_region_servers, conf_path); +} +void TestUtil::StopMiniCluster() { mini->StopCluster(); } + +jobject TestUtil::CreateTable(std::string tblNam, std::string familyName) { + return mini->CreateTable(tblNam, familyName); +} +jobject TestUtil::CreateTable(std::string tblNam, std::string familyName, std::string key1, + std::string k2) { + return mini->CreateTable(tblNam, familyName, key1, k2); +} +jobject TestUtil::TablePut(std::string table, std::string row, std::string fam, std::string col, + std::string value) { + return mini->TablePut(table, row, fam, col, value); +} diff --git a/hbase-native-client/test-util/test-util.h b/hbase-native-client/test-util/test-util.h index cc35511..aec1a4e 100644 --- a/hbase-native-client/test-util/test-util.h +++ b/hbase-native-client/test-util/test-util.h @@ -22,9 +22,10 @@ #include #include +#include #include - #include "core/configuration.h" +#include "test-util/mini-cluster.h" namespace hbase { /** @@ -36,6 +37,10 @@ class TestUtil { * Creating a TestUtil will spin up a cluster. */ TestUtil(); + /** + * Creating a TestUtil will spin up a cluster with numRegionServers region servers. + */ + TestUtil(int numRegionServers, const char *confPath); /** * Destroying a TestUtil will spin down a cluster and remove the test dir. @@ -62,8 +67,22 @@ class TestUtil { */ std::shared_ptr conf() const { return conf_; } + /** + * Starts mini hbase cluster with specified number of region servers + */ + jobject StartMiniCluster(int num_region_servers); + + void StopMiniCluster(); + jobject CreateTable(std::string tblNam, std::string familyName); + jobject CreateTable(std::string tblNam, std::string familyName, std::string key1, std::string k2); + jobject TablePut(std::string table, std::string row, std::string fam, std::string col, + std::string value); + private: + std::unique_ptr mini; folly::test::TemporaryDirectory temp_dir_; + int numRegionServers = 2; + std::string conf_path; std::shared_ptr conf_ = std::make_shared(); }; } // namespace hbase diff --git a/hbase-native-client/third-party/BUCK b/hbase-native-client/third-party/BUCK index f37eb4e..a55a6fb 100644 --- a/hbase-native-client/third-party/BUCK +++ b/hbase-native-client/third-party/BUCK @@ -112,4 +112,6 @@ cxx_library( ('googletest/googlemock', 'src/*.cc'), ]), exported_deps=dynamic_rules, - visibility=['PUBLIC',],) + visibility=[ + 'PUBLIC', + ],) diff --git a/hbase-native-client/utils/BUCK b/hbase-native-client/utils/BUCK index 796f2f5..4d54b85 100644 --- a/hbase-native-client/utils/BUCK +++ b/hbase-native-client/utils/BUCK @@ -18,12 +18,22 @@ cxx_library( name="utils", exported_headers=["user-util.h", "version.h"], - srcs=["user-util.cc",], - deps=['//third-party:folly',], + srcs=[ + "user-util.cc", + ], + deps=[ + '//third-party:folly', + ], tests=[":user-util-test"], - visibility=['PUBLIC',], + visibility=[ + 'PUBLIC', + ], compiler_flags=['-Weffc++'],) cxx_test( name="user-util-test", - srcs=["user-util-test.cc",], - deps=[":utils",],) + srcs=[ + "user-util-test.cc", + ], + deps=[ + ":utils", + ],) diff --git a/hbase-native-client/test-util/mini-cluster.cc b/hbase-native-client/test-util/mini-cluster.cc new file mode 100644 index 0000000..c466eaa --- /dev/null +++ b/hbase-native-client/test-util/mini-cluster.cc @@ -0,0 +1,347 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "test-util/mini-cluster.h" +#include +#include +#include +#include +#include +#include "core/client.h" +#include "core/configuration.h" +#include "core/get.h" +#include "core/hbase_configuration_loader.h" +#include "core/result.h" +#include "core/table.h" +#include "serde/table-name.h" + +using hbase::MiniCluster; + +JNIEnv *MiniCluster::CreateVM(JavaVM **jvm) { + JavaVMInitArgs args; + JavaVMOption jvm_options; + args.version = JNI_VERSION_1_6; + args.nOptions = 1; + char *classpath = getenv("CLASSPATH"); + std::string clspath; + if (classpath == NULL || strstr(classpath, "-tests.jar") == NULL) { + std::string clsPathFilePath("../target/cached_classpath.txt"); + std::ifstream fd(clsPathFilePath); + std::string prefix(""); + if (fd.is_open()) { + if (classpath == NULL) { + LOG(INFO) << "got empty classpath"; + } else { + // prefix bootstrapper.jar + prefix.assign(classpath); + } + std::string line; + if (getline(fd, line)) { + clspath = prefix + ":" + line; + int ret = setenv("CLASSPATH", clspath.c_str(), 1); + LOG(INFO) << "set clspath " << ret; + } else { + LOG(INFO) << "nothing read from " << clsPathFilePath; + exit(-1); + } + } else { + LOG(INFO) << "nothing read from " << clsPathFilePath; + exit(-1); + } + fd.close(); + } + auto options = std::string {"-Djava.class.path="} + clspath; + jvm_options.optionString = const_cast(options.c_str()); + args.options = &jvm_options; + args.ignoreUnrecognized = 0; + int rv; + rv = JNI_CreateJavaVM(jvm, reinterpret_cast(&env_), &args); + if (rv < 0 || !env_) { + LOG(INFO) << "Unable to Launch JVM " << rv; + } else { + LOG(INFO) << "Launched JVM! " << options; + } + return env_; +} + +void MiniCluster::WriteConf(jobject conf, const char *filepath) { + jclass class_fdesc = env_->FindClass("java/io/FileDescriptor"); + // construct a new FileDescriptor + jmethodID const_fdesc = env_->GetMethodID(class_fdesc, "", "()V"); + + jobject file = env_->NewObject(class_fdesc, const_fdesc); + jfieldID field_fd = env_->GetFieldID(class_fdesc, "fd", "I"); + + int fd = open(filepath, O_RDWR | O_NONBLOCK | O_CREAT, S_IRWXU); + if (fd < 0) { + LOG(INFO) << "Couldn't open file " << filepath; + exit(-1); + } + env_->SetIntField(file, field_fd, fd); + + jclass cls_outstream = env_->FindClass("java/io/FileOutputStream"); + jmethodID ctor_stream = env_->GetMethodID(cls_outstream, "", "(Ljava/io/FileDescriptor;)V"); + if (ctor_stream == NULL) { + LOG(INFO) << "Couldn't get ctor for FileOutputStream"; + exit(-1); + } + jobject file_outstream = env_->NewObject(cls_outstream, ctor_stream, file); + if (file_outstream == NULL) { + LOG(INFO) << "Couldn't create FileOutputStream"; + exit(-1); + } + jmethodID writeXmlMid = env_->GetMethodID(confClass_, "writeXml", "(Ljava/io/OutputStream;)V"); + env_->CallObjectMethod(conf, writeXmlMid, file_outstream); +} +void MiniCluster::Setup() { + jmethodID constructor; + pthread_mutex_lock(&count_mutex); + if (env_ == NULL) { + env_ = CreateVM(&jvm); + if (env_ == NULL) { + exit(-1); + } + testingUtilClass_ = env_->FindClass("org/apache/hadoop/hbase/HBaseTestingUtility"); + // this should be converted to a globalref I think to avoid the underlying java obj getting + // GC'ed + if (testingUtilClass_ == NULL) { + LOG(INFO) << "Couldn't find class HBaseTestingUtility"; + exit(-1); + } + jmethodID mid = env_->GetStaticMethodID(testingUtilClass_, "createLocalHTU", + "()Lorg/apache/hadoop/hbase/HBaseTestingUtility;"); + htu_ = env_->CallStaticObjectMethod(testingUtilClass_, mid); + // this should be converted to a globalref I think to avoid the underlying java obj getting + // GC'ed + if (htu_ == NULL) { + LOG(INFO) << "Couldn't invoke method createLocalHTU in HBaseTestingUtility"; + exit(-1); + } + getConnMid_ = env_->GetMethodID(testingUtilClass_, "getConnection", + "()Lorg/apache/hadoop/hbase/client/Connection;"); + jclass connClass = env_->FindClass("org/apache/hadoop/hbase/client/Connection"); + getAdminMid_ = + env_->GetMethodID(connClass, "getAdmin", "()Lorg/apache/hadoop/hbase/client/Admin;"); + getTableMid_ = env_->GetMethodID( + connClass, "getTable", + "(Lorg/apache/hadoop/hbase/TableName;)Lorg/apache/hadoop/hbase/client/Table;"); + if (getTableMid_ == NULL) { + LOG(INFO) << "Couldn't find getConnection"; + exit(-1); + } + jclass adminClass = env_->FindClass("org/apache/hadoop/hbase/client/Admin"); + moveMid_ = env_->GetMethodID(adminClass, "move", "([B[B)V"); + if (moveMid_ == NULL) { + LOG(INFO) << "Couldn't find move"; + exit(-1); + } + createTableMid_ = env_->GetMethodID(testingUtilClass_, "createTable", + "(Lorg/apache/hadoop/hbase/TableName;Ljava/lang/String;)Lorg/" + "apache/hadoop/hbase/client/Table;"); + createTableWithSplitMid_ = env_->GetMethodID( + testingUtilClass_, "createTable", + "(Lorg/apache/hadoop/hbase/TableName;[[B[[B)Lorg/apache/hadoop/hbase/client/Table;"); + if (createTableWithSplitMid_ == NULL) { + LOG(INFO) << "Couldn't find method createTable with split"; + exit(-1); + } + + tableNameClass_ = env_->FindClass("org/apache/hadoop/hbase/TableName"); + tblNameValueOfMid_ = env_->GetStaticMethodID( + tableNameClass_, "valueOf", "(Ljava/lang/String;)Lorg/apache/hadoop/hbase/TableName;"); + if (tblNameValueOfMid_ == NULL) { + LOG(INFO) << "Couldn't find method valueOf in TableName"; + exit(-1); + } + jclass hbaseMiniClusterClass = env_->FindClass("org/apache/hadoop/hbase/MiniHBaseCluster"); + stopRSMid_ = + env_->GetMethodID(hbaseMiniClusterClass, "stopRegionServer", + "(I)Lorg/apache/hadoop/hbase/util/JVMClusterUtil$RegionServerThread;"); + getConfMid_ = env_->GetMethodID(hbaseMiniClusterClass, "getConfiguration", + "()Lorg/apache/hadoop/conf/Configuration;"); + + confClass_ = env_->FindClass("org/apache/hadoop/conf/Configuration"); + setConfMid_ = env_->GetMethodID(confClass_, "set", "(Ljava/lang/String;Ljava/lang/String;)V"); + if (setConfMid_ == NULL) { + LOG(INFO) << "Couldn't find method getConf in MiniHBaseCluster"; + exit(-1); + } + confGetMid_ = env_->GetMethodID(confClass_, "get", "(Ljava/lang/String;)Ljava/lang/String;"); + + jclass tableClass = env_->FindClass("org/apache/hadoop/hbase/client/Table"); + putMid_ = env_->GetMethodID(tableClass, "put", "(Lorg/apache/hadoop/hbase/client/Put;)V"); + jclass connFactoryClass = env_->FindClass("org/apache/hadoop/hbase/client/ConnectionFactory"); + createConnMid_ = env_->GetStaticMethodID(connFactoryClass, "createConnection", + "()Lorg/apache/hadoop/hbase/client/Connection;"); + if (createConnMid_ == NULL) { + LOG(INFO) << "Couldn't find createConnection"; + exit(-1); + } + putClass_ = env_->FindClass("org/apache/hadoop/hbase/client/Put"); + putCtor = env_->GetMethodID(putClass_, "", "([B)V"); + addColMid_ = + env_->GetMethodID(putClass_, "addColumn", "([B[B[B)Lorg/apache/hadoop/hbase/client/Put;"); + if (addColMid_ == NULL) { + LOG(INFO) << "Couldn't find method addColumn"; + exit(-1); + } + } + pthread_mutex_unlock(&count_mutex); +} + +jobject MiniCluster::htu() { + Setup(); + return htu_; +} + +JNIEnv *MiniCluster::env() { + Setup(); + return env_; +} +// converts C char* to Java byte[] +jbyteArray MiniCluster::StrToByteChar(const char *str) { + char *p = const_cast(str); + int n = 0; + while (*p++) { + n++; + } + if (n == 0) return NULL; + jbyteArray arr = env_->NewByteArray(n); + env_->SetByteArrayRegion(arr, 0, n, (jbyte *)str); + return arr; +} + +jobject MiniCluster::CreateTable(std::string tblNam, std::string familyName) { + jstring tblNameStr = env_->NewStringUTF(tblNam.c_str()); + jobject tblName = env_->CallStaticObjectMethod(tableNameClass_, tblNameValueOfMid_, tblNameStr); + jstring famStr = env_->NewStringUTF(familyName.c_str()); + jobject tbl = env_->CallObjectMethod(htu_, createTableMid_, tblName, famStr); + return tbl; +} +jobject MiniCluster::CreateTable(std::string tblNam, std::string familyName, std::string key1, + std::string key2) { + jstring tblNameStr = env_->NewStringUTF(tblNam.c_str()); + jobject tblName = env_->CallStaticObjectMethod(tableNameClass_, tblNameValueOfMid_, tblNameStr); + jclass arrayElemType = env_->FindClass("[B"); + + jobjectArray famArray = env_->NewObjectArray(1, arrayElemType, env_->NewByteArray(1)); + env_->SetObjectArrayElement(famArray, 0, StrToByteChar(familyName.c_str())); + + int len = 2; + if (key2.empty()) len = 1; + jobjectArray keyArray = env_->NewObjectArray(len, arrayElemType, env_->NewByteArray(1)); + + env_->SetObjectArrayElement(keyArray, 0, StrToByteChar(key1.c_str())); + if (!key2.empty()) { + env_->SetObjectArrayElement(keyArray, 1, StrToByteChar(key2.c_str())); + } + jobject tbl = env_->CallObjectMethod(htu_, createTableWithSplitMid_, tblName, famArray, keyArray); + return tbl; +} + +jobject MiniCluster::StopRegionServer(jobject cluster, int idx) { + env(); + return env_->CallObjectMethod(cluster, stopRSMid_, (jint)idx); +} + +// returns the Configuration for the cluster +jobject MiniCluster::GetConf() { + env(); + return env_->CallObjectMethod(cluster_, getConfMid_); +} +// return the Admin instance for the local cluster +jobject MiniCluster::admin() { + env(); + jobject conn = env_->CallObjectMethod(htu(), getConnMid_); + jobject admin = env_->CallObjectMethod(conn, getAdminMid_); + return admin; +} + +jobject MiniCluster::TablePut(const std::string table, const std::string row, const std::string fam, + const std::string col, const std::string value) { + env(); + jobject conn = env_->CallObjectMethod(htu(), getConnMid_); + jobject put = env_->NewObject(putClass_, putCtor, StrToByteChar(row.c_str())); + if (put == 0) { + LOG(INFO) << "Couldn't create Put"; + exit(-1); + } + env_->CallObjectMethod(put, addColMid_, StrToByteChar(fam.c_str()), StrToByteChar(col.c_str()), + StrToByteChar(value.c_str())); + jobject tblName = + env_->CallStaticObjectMethod(tableNameClass_, tblNameValueOfMid_, + env_->NewStringUTF(table.c_str())); + jobject tableObj = env_->CallObjectMethod(conn, getTableMid_, tblName); + env_->CallObjectMethod(tableObj, putMid_, put); + return tableObj; +} + +// moves region to server +void MiniCluster::MoveRegion(std::string region, std::string server) { + jobject admin_ = admin(); + env_->CallObjectMethod(admin_, moveMid_, StrToByteChar(region.c_str()), + StrToByteChar(server.c_str())); +} + +jobject MiniCluster::StartCluster(int numRegionServers, std::string conf_path) { + env(); + jmethodID mid = env_->GetMethodID(testingUtilClass_, "startMiniCluster", + "(I)Lorg/apache/hadoop/hbase/MiniHBaseCluster;"); + if (mid == 0) { + LOG(INFO) << "Couldn't find method startMiniCluster in the class HBaseTestingUtility"; + exit(-1); + } + cluster_ = env_->CallObjectMethod(htu(), mid, (jint)numRegionServers); + jobject conf = GetConf(); + jstring jport = (jstring)env_->CallObjectMethod( + conf, confGetMid_, env_->NewStringUTF("hbase.zookeeper.property.clientPort")); + const char *port = env_->GetStringUTFChars(jport, 0); + LOG(INFO) << "retrieved port " << port; + std::string quorum("localhost:"); + env_->CallObjectMethod(conf, setConfMid_, env_->NewStringUTF("hbase.zookeeper.quorum"), + env_->NewStringUTF((quorum + port).c_str())); + if (!conf_path.empty()) { + // Directory will be created if not present + if (!boost::filesystem::exists(conf_path)) { + boost::filesystem::create_directories(conf_path); + } + WriteConf(conf, (conf_path + "/hbase-site.xml").c_str()); + } + return cluster_; +} + +void MiniCluster::StopCluster() { + env(); + jmethodID mid = env_->GetMethodID(testingUtilClass_, "shutdownMiniCluster", "()V"); + env_->CallVoidMethod(htu(), mid); + if (jvm != NULL) { + jvm->DestroyJavaVM(); + jvm = NULL; + } +} + +const std::string MiniCluster::GetConfValue(std::string key) { + jobject conf = GetConf(); + LOG(INFO) << "retrieving " << key; + jstring jval = (jstring)env_->CallObjectMethod(conf, confGetMid_, + env_->NewStringUTF(key.c_str())); + const char *val = env_->GetStringUTFChars(jval, 0); + LOG(INFO) << "Got string " << val; + return val; +} diff --git a/hbase-native-client/test-util/mini-cluster.h b/hbase-native-client/test-util/mini-cluster.h new file mode 100644 index 0000000..7ecf3fa --- /dev/null +++ b/hbase-native-client/test-util/mini-cluster.h @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +#pragma once + +#include +#include +#include "core/client.h" +#include "core/configuration.h" +#include "core/get.h" +#include "core/hbase_configuration_loader.h" +#include "core/result.h" +#include "core/table.h" +#include "jni.h" +#include "serde/table-name.h" +namespace hbase { +class MiniCluster { + public: + jobject StartCluster(int numRegionServers, std::string conf_path); + void StopCluster(); + jobject CreateTable(std::string tblNam, std::string familyName); + jobject CreateTable(std::string tblNam, std::string familyName, std::string key1, std::string k2); + jobject StopRegionServer(jobject cluster, int idx); + + // moves region to server + void MoveRegion(std::string region, std::string server); + // returns the Configuration instance for the cluster + jobject GetConf(); + // returns the value for config key retrieved from cluster + const std::string GetConfValue(std::string key); + // Does Put into table for family fam, qualifier col with value + jobject TablePut(const std::string table, const std::string row, const std::string fam, + const std::string col, const std::string value); + + private: + JNIEnv *env_; + jclass testingUtilClass_; + jclass tableNameClass_; + jclass putClass_; + jclass confClass_; + jmethodID stopRSMid_; + jmethodID getConfMid_; + jmethodID setConfMid_; + jmethodID tblNameValueOfMid_; + jmethodID createTableMid_; + jmethodID createTableWithSplitMid_; + jmethodID putMid_; + jmethodID putCtor; + jmethodID addColMid_; + jmethodID createConnMid_; + jmethodID getConnMid_; + jmethodID getTableMid_; + jmethodID confGetMid_; + jmethodID getAdminMid_; + jmethodID moveMid_; + jmethodID strCtorMid_; + jobject htu_; + jobject cluster_; + pthread_mutex_t count_mutex; + JavaVM *jvm; + JNIEnv *CreateVM(JavaVM **jvm); + void WriteConf(jobject conf, const char *filepath); + void Setup(); + jobject htu(); + JNIEnv *env(); + jbyteArray StrToByteChar(const char *str); + jobject admin(); +}; +} /*namespace hbase*/