diff --git a/hbase-native-client/Makefile b/hbase-native-client/Makefile index 84ae556..9eec714 100644 --- a/hbase-native-client/Makefile +++ b/hbase-native-client/Makefile @@ -26,13 +26,13 @@ MODULES = connection core serde test-util utils security SRC_DIR = $(MODULES) DEBUG_BUILD_DIR = $(addprefix $(DEBUG_PATH)/,$(MODULES)) RELEASE_BUILD_DIR = $(addprefix $(RELEASE_PATH)/,$(MODULES)) -INCLUDE_DIR = . build/ +INCLUDE_DIR = . build/ /usr/lib/jvm/java-8-openjdk-amd64/include/ /usr/lib/jvm/java-8-openjdk-amd64/include/linux #flags to pass to the CPP compiler & linker CPPFLAGS_DEBUG = -D_GLIBCXX_USE_CXX11_ABI=0 -g -Wall -std=c++14 -pedantic -fPIC CPPFLAGS_RELEASE = -D_GLIBCXX_USE_CXX11_ABI=0 -DNDEBUG -O2 -Wall -std=c++14 -pedantic -fPIC LDFLAGS = -lprotobuf -lzookeeper_mt -lsasl2 -lfolly -lwangle -LINKFLAG = -shared +LINKFLAG = -shared -L/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server -ljvm #define list of source files and object files ALLSRC = $(foreach sdir,$(SRC_DIR),$(wildcard $(sdir)/*.cc)) diff --git a/hbase-native-client/core/client-test.cc b/hbase-native-client/core/client-test.cc index 0a45fff..469f851 100644 --- a/hbase-native-client/core/client-test.cc +++ b/hbase-native-client/core/client-test.cc @@ -27,7 +27,7 @@ #include "serde/table-name.h" #include "test-util/test-util.h" -class ClientTest { +class ClientTest : public ::testing::Test { public: const static std::string kDefHBaseConfPath; @@ -57,10 +57,17 @@ class ClientTest { static void CreateHBaseConfWithEnv() { // Creating Empty Config Files so that we dont get a Configuration exception @Client CreateHBaseConf(kDefHBaseConfPath, kHBaseDefaultXml, kHBaseXmlData); - CreateHBaseConf(kDefHBaseConfPath, kHBaseSiteXml, kHBaseXmlData); + // the hbase-site.xml would be persisted by MiniCluster + //CreateHBaseConf(kDefHBaseConfPath, kHBaseSiteXml, kHBaseXmlData); setenv("HBASE_CONF", kDefHBaseConfPath.c_str(), 1); } + static std::unique_ptr test_util; + + static void SetUpTestCase() { + test_util = std::make_unique(2, ClientTest::kDefHBaseConfPath.c_str()); + } }; +std::unique_ptr ClientTest::test_util = nullptr; const std::string ClientTest::kDefHBaseConfPath("./build/test-data/client-test/conf/"); @@ -109,16 +116,15 @@ TEST(Client, DefaultConfiguration) { client.Close(); } -TEST(Client, Get) { +TEST_F(ClientTest, Get) { // Remove already configured env if present. unsetenv("HBASE_CONF"); ClientTest::CreateHBaseConfWithEnv(); // Using TestUtil to populate test data - hbase::TestUtil *test_util = new hbase::TestUtil(); - test_util->RunShellCmd( - "create 't', 'd'; put 't', 'test2', 'd:2', 'value2'; put 't', 'test2', 'd:extra', 'value for " - "extra'"); + ClientTest::test_util->CreateTable("t", "d"); + ClientTest::test_util->TablePut("t", "test2", "d", "2", "value2"); + ClientTest::test_util->TablePut("t", "test2", "d", "extra", "value for extra"); // Create TableName and Row to be fetched from HBase auto tn = folly::to("t"); @@ -145,7 +151,7 @@ TEST(Client, Get) { // The connection stays open and we don't want that. // So we are stopping the connection. // We can remove this once we have fixed the folly part - delete test_util; + //delete test_util; // Test the values, should be same as in put executed on hbase shell ASSERT_TRUE(!result->IsEmpty()) << "Result shouldn't be empty."; @@ -163,7 +169,7 @@ TEST(Client, GetForNonExistentTable) { ClientTest::CreateHBaseConfWithEnv(); // Using TestUtil to populate test data - hbase::TestUtil *test_util = new hbase::TestUtil(); + //hbase::TestUtil *test_util = new hbase::TestUtil(2, ClientTest::kDefHBaseConfPath.c_str()); // Create TableName and Row to be fetched from HBase auto tn = folly::to("t_not_exists"); @@ -190,20 +196,20 @@ TEST(Client, GetForNonExistentTable) { // The connection stays open and we don't want that. // So we are stopping the connection. // We can remove this once we have fixed the folly part - delete test_util; + //delete test_util; table->Close(); client.Close(); } -TEST(Client, GetForNonExistentRow) { +TEST_F(ClientTest, GetForNonExistentRow) { // Remove already configured env if present. unsetenv("HBASE_CONF"); ClientTest::CreateHBaseConfWithEnv(); // Using TestUtil to populate test data - hbase::TestUtil *test_util = new hbase::TestUtil(); - test_util->RunShellCmd("create 't_exists', 'd'"); + //hbase::TestUtil *test_util = new hbase::TestUtil(2, ClientTest::kDefHBaseConfPath.c_str()); + ClientTest::test_util->CreateTable("t_exists", "d"); // Create TableName and Row to be fetched from HBase auto tn = folly::to("t_exists"); @@ -231,7 +237,7 @@ TEST(Client, GetForNonExistentRow) { // The connection stays open and we don't want that. // So we are stopping the connection. // We can remove this once we have fixed the folly part - delete test_util; + //delete test_util; table->Close(); client.Close(); diff --git a/hbase-native-client/core/filter-test.cc b/hbase-native-client/core/filter-test.cc index ff683b6..d073c8a 100644 --- a/hbase-native-client/core/filter-test.cc +++ b/hbase-native-client/core/filter-test.cc @@ -39,7 +39,16 @@ using hbase::Comparator; class FilterTest : public ::testing::Test { protected: - static void SetUpTestCase() { test_util_ = std::make_unique(); } + static void SetUpTestCase() { + unsetenv("HBASE_CONF"); + std::string confDir("./build/test-data/filter-test/conf/"); + // Directory will be created if not present + if (!boost::filesystem::exists(confDir)) { + boost::filesystem::create_directories(confDir); + } + setenv("HBASE_CONF", confDir.c_str(), 1); + test_util_ = std::make_unique(2, confDir.c_str()); + } static void TearDownTestCase() { test_util_.release(); } @@ -53,9 +62,14 @@ std::unique_ptr FilterTest::test_util_ = nullptr; TEST_F(FilterTest, GetWithColumnPrefixFilter) { // write row1 with 3 columns (column_1, column_2, and foo_column) - FilterTest::test_util_->RunShellCmd( + FilterTest::test_util_->CreateTable("t", "d"); + FilterTest::test_util_->TablePut("t", "row1", "d", "column_1", "value1"); + FilterTest::test_util_->TablePut("t", "row1", "d", "column_2", "value2"); + FilterTest::test_util_->TablePut("t", "row1", "d", "foo_column", "value3"); + + /*FilterTest::test_util_->RunShellCmd( "create 't', 'd'; put 't', 'row1', 'd:column_1', 'value1'; put 't', 'row1', 'd:column_2', " - "'value2'; put 't', 'row1', 'd:foo_column', 'value3'"); + "'value2'; put 't', 'row1', 'd:foo_column', 'value3'"); */ // Create TableName and Row to be fetched from HBase auto tn = folly::to("t"); @@ -69,8 +83,11 @@ TEST_F(FilterTest, GetWithColumnPrefixFilter) { get_one.SetFilter(FilterFactory::ColumnPrefixFilter("foo_")); get_two.SetFilter(FilterFactory::ColumnPrefixFilter("column_")); + // Create Configuration + hbase::HBaseConfigurationLoader loader; + auto conf = loader.LoadDefaultResources(); // Create a client - hbase::Client client(Configuration{}); + hbase::Client client(conf.value()); // Get connection to HBase Table auto table = client.Table(tn); @@ -101,9 +118,13 @@ TEST_F(FilterTest, GetWithColumnPrefixFilter) { TEST_F(FilterTest, GetWithQualifierFilter) { // write row1 with 3 columns (a,b,c) - FilterTest::test_util_->RunShellCmd( + FilterTest::test_util_->CreateTable("t1", "d"); + FilterTest::test_util_->TablePut("t1", "row1", "d", "a", "value1"); + FilterTest::test_util_->TablePut("t1", "row1", "d", "b", "value2"); + FilterTest::test_util_->TablePut("t1", "row1", "d", "c", "value3"); + /*FilterTest::test_util_->RunShellCmd( "create 't1', 'd'; put 't1', 'row1', 'd:a', 'value1'; put 't1', 'row1', 'd:b', " - "'value2'; put 't1', 'row1', 'd:c', 'value3'"); + "'value2'; put 't1', 'row1', 'd:c', 'value3'");*/ // Create TableName and Row to be fetched from HBase auto tn = folly::to("t1"); @@ -114,8 +135,11 @@ TEST_F(FilterTest, GetWithQualifierFilter) { get.SetFilter(FilterFactory::QualifierFilter(CompareType::GREATER_OR_EQUAL, *ComparatorFactory::BinaryComparator("b"))); + // Create Configuration + hbase::HBaseConfigurationLoader loader; + auto conf = loader.LoadDefaultResources(); // Create a client - hbase::Client client(Configuration{}); + hbase::Client client(conf.value()); // Get connection to HBase Table auto table = client.Table(tn); diff --git a/hbase-native-client/core/location-cache-test.cc b/hbase-native-client/core/location-cache-test.cc index 9a778c6..7b2f228 100644 --- a/hbase-native-client/core/location-cache-test.cc +++ b/hbase-native-client/core/location-cache-test.cc @@ -59,7 +59,7 @@ TEST(LocationCacheTest, TestGetRegionLocation) { auto tn = folly::to("t"); auto row = "test"; ASSERT_ANY_THROW(cache.LocateFromMeta(tn, row).get(milliseconds(1000))); - test_util.RunShellCmd("create 't', 'd'"); + test_util.CreateTable("t", "d"); auto loc = cache.LocateFromMeta(tn, row).get(milliseconds(1000)); ASSERT_TRUE(loc != nullptr); cpu->stop(); @@ -83,8 +83,7 @@ TEST(LocationCacheTest, TestCaching) { // test location pulled from meta gets cached ASSERT_ANY_THROW(cache.LocateRegion(tn_1, row_a).get(milliseconds(1000))); ASSERT_ANY_THROW(cache.LocateFromMeta(tn_1, row_a).get(milliseconds(1000))); - - test_util.RunShellCmd("create 't1', 'd'"); + test_util.CreateTable("t1", "d"); ASSERT_FALSE(cache.IsLocationCached(tn_1, row_a)); auto loc = cache.LocateRegion(tn_1, row_a).get(milliseconds(1000)); @@ -92,7 +91,7 @@ TEST(LocationCacheTest, TestCaching) { ASSERT_EQ(loc, cache.GetCachedLocation(tn_1, row_a)); // test with two regions - test_util.RunShellCmd("create 't2', 'd', SPLITS => ['b']"); + test_util.CreateTable("t2", "d", "b", NULL); ASSERT_FALSE(cache.IsLocationCached(tn_2, "a")); loc = cache.LocateRegion(tn_2, "a").get(milliseconds(1000)); @@ -107,7 +106,7 @@ TEST(LocationCacheTest, TestCaching) { ASSERT_EQ(loc, cache.GetCachedLocation(tn_2, "ba")); // test with three regions - test_util.RunShellCmd("create 't3', 'd', SPLITS => ['b', 'c']"); + test_util.CreateTable("t3", "d", "b", "c"); ASSERT_FALSE(cache.IsLocationCached(tn_3, "c")); ASSERT_FALSE(cache.IsLocationCached(tn_3, "ca")); diff --git a/hbase-native-client/core/location-cache.cc b/hbase-native-client/core/location-cache.cc index da9f64a..441f8e2 100644 --- a/hbase-native-client/core/location-cache.cc +++ b/hbase-native-client/core/location-cache.cc @@ -106,7 +106,8 @@ ServerName LocationCache::ReadMetaLocation() { reinterpret_cast(buf->writableData()), &len, nullptr); if (zk_result != ZOK || len < 9) { LOG(ERROR) << "Error getting meta location."; - throw runtime_error("Error getting meta location"); + throw runtime_error("Error getting meta location. Quorum " + + conf_->Get(kHBaseZookeeperQuorum_, "")); } buf->append(len); diff --git a/hbase-native-client/test-util/BUCK b/hbase-native-client/test-util/BUCK index c6e41f1..dad936d 100644 --- a/hbase-native-client/test-util/BUCK +++ b/hbase-native-client/test-util/BUCK @@ -17,13 +17,18 @@ cxx_library(name="test-util", exported_headers=[ - "test-util.h", + "test-util.h","mini-cluster.h" ], - srcs=["test-util.cc"], + srcs=["test-util.cc","mini-cluster.cc"], deps=[ "//third-party:folly", - "//core:core" + "//core:core", ], + preprocessor_flags= ['-I', '/usr/lib/jvm/java-8-openjdk-amd64/include/', '-I', '/usr/lib/jvm/java-8-openjdk-amd64/include/linux'], + exported_preprocessor_flags= ['-I', '/usr/lib/jvm/java-8-openjdk-amd64/include/', '-I', '/usr/lib/jvm/java-8-openjdk-amd64/include/linux'], + compiler_flags = ['-I', '/usr/lib/jvm/java-8-openjdk-amd64/include/', '-I', '/usr/lib/jvm/java-8-openjdk-amd64/include/linux'], + linker_flags = ['-ljvm', '-L/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server'], + exported_linker_flags = ['-ljvm', '-L/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server', '-Wl,-rpath=/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server'], visibility=[ 'PUBLIC', ], diff --git a/hbase-native-client/test-util/test-util.cc b/hbase-native-client/test-util/test-util.cc index e355bdf..35060fe 100644 --- a/hbase-native-client/test-util/test-util.cc +++ b/hbase-native-client/test-util/test-util.cc @@ -18,6 +18,7 @@ */ #include "test-util/test-util.h" +#include #include @@ -41,16 +42,19 @@ std::string TestUtil::RandString(int len) { return s; } -TestUtil::TestUtil() : temp_dir_(TestUtil::RandString()) { +TestUtil::TestUtil() : TestUtil::TestUtil(2, "") {} + +TestUtil::TestUtil(int servers, const char *confPath) : + temp_dir_(TestUtil::RandString()), numRegionServers(servers), conf_path(confPath) { auto p = temp_dir_.path().string(); - auto cmd = std::string{"bin/start-local-hbase.sh " + p}; - auto res_code = std::system(cmd.c_str()); - CHECK_EQ(res_code, 0); + jobject cluster = StartMiniCluster(2); + std::string quorum("localhost:"); + const char* port = mini->GetConf(cluster, "hbase.zookeeper.property.clientPort"); + conf()->Set("hbase.zookeeper.quorum", quorum + port); } TestUtil::~TestUtil() { - auto res_code = std::system("bin/stop-local-hbase.sh"); - CHECK_EQ(res_code, 0); + StopMiniCluster(); } void TestUtil::RunShellCmd(const std::string &command) { @@ -58,3 +62,20 @@ void TestUtil::RunShellCmd(const std::string &command) { auto res_code = std::system(cmd_string.c_str()); CHECK_EQ(res_code, 0); } + +jobject TestUtil::StartMiniCluster(int num_region_servers) { + mini = new MiniCluster(); + return mini->StartCluster(num_region_servers, conf_path); +} +void TestUtil::StopMiniCluster() { + mini->StopCluster(); +} +jobject TestUtil::CreateTable(char *tblNam, char *familyName) { + return mini->CreateTable(tblNam, familyName); +} +jobject TestUtil::CreateTable(char *tblNam, char *familyName, char* key1, char*k2) { + return mini->CreateTable(tblNam, familyName, key1, k2); +} +jobject TestUtil::TablePut(char *table, char *row, char *fam, char *col, char *value) { + return mini->TablePut(table, row, fam, col, value); +} \ No newline at end of file diff --git a/hbase-native-client/test-util/test-util.h b/hbase-native-client/test-util/test-util.h index cc35511..0648733 100644 --- a/hbase-native-client/test-util/test-util.h +++ b/hbase-native-client/test-util/test-util.h @@ -23,7 +23,7 @@ #include #include - +#include "test-util/mini-cluster.h" #include "core/configuration.h" namespace hbase { @@ -36,6 +36,10 @@ class TestUtil { * Creating a TestUtil will spin up a cluster. */ TestUtil(); + /** + * Creating a TestUtil will spin up a cluster with numRegionServers region servers. + */ + TestUtil(int numRegionServers, const char *confPath); /** * Destroying a TestUtil will spin down a cluster and remove the test dir. @@ -62,8 +66,21 @@ class TestUtil { */ std::shared_ptr conf() const { return conf_; } + /** + * Starts mini hbase cluster with specified number of region servers + */ + jobject StartMiniCluster(int num_region_servers); + + void StopMiniCluster(); + jobject CreateTable(char *tblNam, char *familyName); + jobject CreateTable(char *tblNam, char *familyName, char* key1, char*k2); + jobject TablePut(char *table, char *row, char *fam, char *col, char *value); + private: + MiniCluster* mini; folly::test::TemporaryDirectory temp_dir_; + int numRegionServers = 2; + std::string conf_path; std::shared_ptr conf_ = std::make_shared(); }; } // namespace hbase diff --git a/hbase-native-client/test-util/mini-cluster.cc b/hbase-native-client/test-util/mini-cluster.cc new file mode 100644 index 0000000..7fe5444 --- /dev/null +++ b/hbase-native-client/test-util/mini-cluster.cc @@ -0,0 +1,358 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "core/client.h" +#include +#include +#include "core/configuration.h" +#include "core/get.h" +#include "core/hbase_configuration_loader.h" +#include "core/result.h" +#include "core/table.h" +#include "serde/table-name.h" +#include +#include +#include "test-util/mini-cluster.h" + +using hbase::MiniCluster; + + + JNIEnv* MiniCluster::CreateVM(JavaVM **jvm) + { + JavaVMInitArgs args; + JavaVMOption jvm_options; + args.version = JNI_VERSION_1_6; + args.nOptions = 1; + char *classpath = getenv("CLASSPATH"); + std::string clspath; + if (classpath == NULL || strstr(classpath, "-tests.jar")==NULL) { + std::string clsPathFilePath("../target/cached_classpath.txt"); + std::ifstream fd(clsPathFilePath); + std::string prefix(""); + if (fd.is_open()) { + if (classpath == NULL) { + LOG(INFO) << "got empty classpath"; + } else { + // prefix bootstrapper.jar + prefix.assign(classpath); + } + std::string line; + if (getline(fd, line)) { + clspath = prefix + ":" + line; + int ret = setenv("CLASSPATH", clspath.c_str(), 1); + LOG(INFO) << "set clspath " << ret; + } else { + LOG(INFO) << "nothing read from " << clsPathFilePath; + exit(-1); + } + } else { + LOG(INFO) << "nothing read from " << clsPathFilePath; + exit(-1); + } + fd.close(); + } + auto options = std::string{"-Djava.class.path="} + clspath; + jvm_options.optionString = const_cast(options.c_str()); + args.options = &jvm_options; + args.ignoreUnrecognized = 0; + int rv; + rv = JNI_CreateJavaVM(jvm, (void**)&env, &args); + if (rv < 0 || !env) { + LOG(INFO) << "Unable to Launch JVM " << rv; + } else { + LOG(INFO) << "Launched JVM! " << options; + } + return env; + } + +void MiniCluster::WriteConf(jobject conf, const char *filepath) +{ + jclass class_fdesc = env->FindClass("java/io/FileDescriptor"); + // construct a new FileDescriptor + jmethodID const_fdesc = env->GetMethodID(class_fdesc, "", "()V"); + + jobject file = env->NewObject(class_fdesc, const_fdesc); + jfieldID field_fd = env->GetFieldID(class_fdesc, "fd", "I"); + + int fd = open(filepath, O_RDWR | O_NONBLOCK | O_CREAT, S_IRWXU); + if (fd < 0) { + LOG(INFO) << "Couldn't open file " << filepath; + exit(-1); + } + env->SetIntField(file, field_fd, fd); + + jclass cls_outstream = env->FindClass("java/io/FileOutputStream"); + jmethodID ctor_stream = env->GetMethodID(cls_outstream, "", + "(Ljava/io/FileDescriptor;)V"); + if (ctor_stream == NULL) { + LOG(INFO) << "Couldn't get ctor for FileOutputStream"; + exit(-1); + } + jobject file_outstream = env->NewObject(cls_outstream, ctor_stream, file); + if (file_outstream == NULL) { + LOG(INFO) << "Couldn't create FileOutputStream"; + exit(-1); + } + jmethodID writeXmlMid = env->GetMethodID(confClass, "writeXml", + "(Ljava/io/OutputStream;)V"); + env->CallObjectMethod(conf, writeXmlMid, file_outstream); +} +void MiniCluster::setup() +{ + jmethodID constructor; + pthread_mutex_lock(&count_mutex); + if (env == NULL) { + env = CreateVM(&jvm); + if (env == NULL) { + exit(-1); + } + testingUtilClass = env->FindClass("org/apache/hadoop/hbase/HBaseTestingUtility"); + //this should be converted to a globalref I think to avoid the underlying java obj getting GC'ed + if (testingUtilClass == NULL) { + LOG(INFO) << "Couldn't find class HBaseTestingUtility"; + exit(-1); + } + jmethodID mid = env->GetStaticMethodID(testingUtilClass, "createLocalHTU", + "()Lorg/apache/hadoop/hbase/HBaseTestingUtility;"); + htu = env->CallStaticObjectMethod(testingUtilClass, mid); + //this should be converted to a globalref I think to avoid the underlying java obj getting GC'ed + if (htu == NULL) { + LOG(INFO) << "Couldn't invoke method createLocalHTU in HBaseTestingUtility"; + exit(-1); + } + getConnMid = env->GetMethodID(testingUtilClass, "getConnection", + "()Lorg/apache/hadoop/hbase/client/Connection;"); + jclass connClass = env->FindClass("org/apache/hadoop/hbase/client/Connection"); + getAdminMid = env->GetMethodID(connClass, "getAdmin", + "()Lorg/apache/hadoop/hbase/client/Admin;"); + getTableMid = env->GetMethodID(connClass, "getTable", + "(Lorg/apache/hadoop/hbase/TableName;)Lorg/apache/hadoop/hbase/client/Table;"); + if (getTableMid == NULL) { + LOG(INFO) << "Couldn't find getConnection"; + exit(-1); + } + jclass adminClass = env->FindClass("org/apache/hadoop/hbase/client/Admin"); + moveMid = env->GetMethodID(adminClass, "move", + "([B[B)V"); + if (moveMid == NULL) { + LOG(INFO) << "Couldn't find move"; + exit(-1); + } + createTableMid = env->GetMethodID(testingUtilClass, "createTable", + "(Lorg/apache/hadoop/hbase/TableName;Ljava/lang/String;)Lorg/apache/hadoop/hbase/client/Table;"); + createTableWithSplitMid = env->GetMethodID(testingUtilClass, "createTable", + "(Lorg/apache/hadoop/hbase/TableName;[[B[[B)Lorg/apache/hadoop/hbase/client/Table;"); + if (createTableWithSplitMid == NULL) { + LOG(INFO) << "Couldn't find method createTable with split"; + exit(-1); + } + + tableNameClass = env->FindClass("org/apache/hadoop/hbase/TableName"); + tblNameValueOfMid = env->GetStaticMethodID(tableNameClass, "valueOf", + "(Ljava/lang/String;)Lorg/apache/hadoop/hbase/TableName;"); + if (tblNameValueOfMid == NULL) { + LOG(INFO) << "Couldn't find method valueOf in TableName"; + exit(-1); + } + jclass hbaseMiniClusterClass = env->FindClass("org/apache/hadoop/hbase/MiniHBaseCluster"); + stopRSMid = env->GetMethodID(hbaseMiniClusterClass, "stopRegionServer", + "(I)Lorg/apache/hadoop/hbase/util/JVMClusterUtil$RegionServerThread;"); + getConfMid = env->GetMethodID(hbaseMiniClusterClass, "getConfiguration", + "()Lorg/apache/hadoop/conf/Configuration;"); + + confClass = env->FindClass("org/apache/hadoop/conf/Configuration"); + setConfMid = env->GetMethodID(confClass, "set", + "(Ljava/lang/String;Ljava/lang/String;)V"); + if (setConfMid == NULL) { + LOG(INFO) << "Couldn't find method getConf in MiniHBaseCluster"; + exit(-1); + } + confGetMid = env->GetMethodID(confClass, "get", + "(Ljava/lang/String;)Ljava/lang/String;"); + + jclass tableClass = env->FindClass("org/apache/hadoop/hbase/client/Table"); + putMid = env->GetMethodID(tableClass, "put", + "(Lorg/apache/hadoop/hbase/client/Put;)V"); + jclass connFactoryClass = env->FindClass("org/apache/hadoop/hbase/client/ConnectionFactory"); + createConnMid = env->GetStaticMethodID(connFactoryClass, "createConnection", + "()Lorg/apache/hadoop/hbase/client/Connection;"); + if (createConnMid == NULL) { + LOG(INFO) << "Couldn't find createConnection"; + exit(-1); + } + putClass = env->FindClass("org/apache/hadoop/hbase/client/Put"); + putCtor = env->GetMethodID(putClass, "", "([B)V"); + addColMid = env->GetMethodID(putClass, "addColumn", + "([B[B[B)Lorg/apache/hadoop/hbase/client/Put;"); + if (addColMid == NULL) { + LOG(INFO) << "Couldn't find method addColumn"; + exit(-1); + } + } + pthread_mutex_unlock(&count_mutex); +} + +jobject MiniCluster::getHTU() +{ + setup(); + return htu; +} + +JNIEnv* MiniCluster::getEnv() +{ + setup(); + return env; +} +// converts C char* to Java byte[] +jbyteArray MiniCluster::str_to_byte_array(char *str) { + char* p = str; + int n = 0; + while (*p++) { + n++; + } + if (n == 0) return NULL; + jbyteArray arr = env->NewByteArray(n); + env->SetByteArrayRegion(arr, 0, n, (jbyte*)str); + return arr; +} + +jobject MiniCluster::CreateTable(char *tblNam, char *familyName) +{ + jstring tblNameStr = env->NewStringUTF(tblNam); + jobject tblName = env->CallStaticObjectMethod(tableNameClass, tblNameValueOfMid, + tblNameStr); + jstring famStr = env->NewStringUTF(familyName); + jobject tbl = env->CallObjectMethod(htu, createTableMid, tblName, famStr); + return tbl; +} +jobject MiniCluster::CreateTable(char *tblNam, char *familyName, char* key1, char*key2) +{ + jstring tblNameStr = env->NewStringUTF(tblNam); + jobject tblName = env->CallStaticObjectMethod(tableNameClass, tblNameValueOfMid, + tblNameStr); + jclass arrayElemType = env->FindClass("[B"); + + jobjectArray famArray = env->NewObjectArray(1, arrayElemType, env->NewByteArray(1)); + env->SetObjectArrayElement(famArray, 0, str_to_byte_array(familyName)); + + int len = 2; + if (key2 == NULL) len = 1; + jobjectArray keyArray = env->NewObjectArray(len, arrayElemType, env->NewByteArray(1)); + + env->SetObjectArrayElement(keyArray, 0, str_to_byte_array(key1)); + if (key2 != NULL) { + env->SetObjectArrayElement(keyArray, 1, str_to_byte_array(key2)); + } + jobject tbl = env->CallObjectMethod(htu, createTableWithSplitMid, tblName, famArray, keyArray); + return tbl; +} +jobject MiniCluster::StopRegionServer(jobject cluster, int idx) +{ + env = getEnv(); + env->CallObjectMethod(cluster, stopRSMid, (jint)idx); +} + +// returns the Configuration for the cluster +jobject MiniCluster::GetConf(jobject cluster) +{ + JNIEnv* env = getEnv(); + return env->CallObjectMethod(cluster, getConfMid); +} +// return the Admin instance for the local cluster +jobject MiniCluster::getAdmin() +{ + JNIEnv* env = getEnv(); + jobject conn = env->CallObjectMethod(getHTU(), getConnMid); + jobject admin = env->CallObjectMethod(conn, getAdminMid); + return admin; +} + +jobject MiniCluster::TablePut(char *table, char *row, char *fam, char *col, char *value) +{ + JNIEnv* env = getEnv(); + jobject conn = env->CallObjectMethod(getHTU(), getConnMid); + jobject put = env->NewObject(putClass, putCtor, str_to_byte_array(row)); + if (put == 0) { + LOG(INFO) << "Couldn't create Put"; + exit(-1); + } + env->CallObjectMethod(put, addColMid, str_to_byte_array(fam), str_to_byte_array(col), + str_to_byte_array(value)); + jobject tblName = env->CallStaticObjectMethod(tableNameClass, tblNameValueOfMid, + env->NewStringUTF(table)); + jobject tableObj = env->CallObjectMethod(conn, getTableMid, tblName); + env->CallObjectMethod(tableObj, putMid, put); + return tableObj; +} + +// moves region to server +void MiniCluster::MoveRegion(char *region, char *server) +{ + jobject admin = getAdmin(); + env->CallObjectMethod(admin, moveMid, str_to_byte_array(region), + str_to_byte_array(server)); +} + +jobject MiniCluster::StartCluster(int numRegionServers, std::string conf_path) +{ + env = getEnv(); + jmethodID mid = env->GetMethodID(testingUtilClass, "startMiniCluster", + "(I)Lorg/apache/hadoop/hbase/MiniHBaseCluster;"); + if (mid == 0) { + LOG(INFO) << "Couldn't find method startMiniCluster in the class HBaseTestingUtility"; + exit(-1); + } + cluster = env->CallObjectMethod(getHTU(), mid, (jint)numRegionServers); + jobject conf = GetConf(cluster); + jstring jport = (jstring)env->CallObjectMethod(conf, confGetMid, + env->NewStringUTF("hbase.zookeeper.property.clientPort")); + const char* port = env->GetStringUTFChars(jport, 0); + LOG(INFO) << "retrieved port " << port; + std::string quorum("localhost:"); + env->CallObjectMethod(conf, setConfMid, env->NewStringUTF("hbase.zookeeper.quorum"), + env->NewStringUTF((quorum+port).c_str())); + //if (conf_path.empty()) conf_path.assign("./build/test-data/client-test/conf"); + WriteConf(conf, (conf_path + "/hbase-site.xml").c_str()); + return cluster; +} + +void MiniCluster::StopCluster() +{ + env = getEnv(); + jmethodID mid = env->GetMethodID(testingUtilClass, + "shutdownMiniCluster","()V"); + env->CallVoidMethod(getHTU(), mid); + if (jvm != NULL) { + //jvm->DestroyJavaVM(); + //jvm = NULL; + } +} + +// cluster is the object returned by StartCluster() +//const char* GetConf(jobject cluster, const std::string& key) +const char* MiniCluster::GetConf(jobject cluster, char* key) +{ + jobject conf = GetConf(cluster); + LOG(INFO) << "retrieving " << key; + jstring jval = (jstring)env->CallObjectMethod(conf, confGetMid, + env->NewStringUTF(key)); + const char* val = env->GetStringUTFChars(jval, 0); + LOG(INFO) << "Got string " << val; + return val; +} diff --git a/hbase-native-client/test-util/mini-cluster.h b/hbase-native-client/test-util/mini-cluster.h new file mode 100644 index 0000000..cc992ae --- /dev/null +++ b/hbase-native-client/test-util/mini-cluster.h @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +#pragma once + +#include "core/client.h" +#include +#include "core/configuration.h" +#include "core/get.h" +#include "core/hbase_configuration_loader.h" +#include "core/result.h" +#include "core/table.h" +#include "serde/table-name.h" +#include "jni.h" +namespace hbase { + class MiniCluster { + public: + jobject StartCluster(int numRegionServers, std::string conf_path); + void StopCluster(); + jobject CreateTable(char *tblNam, char *familyName); + jobject CreateTable(char *tblNam, char *familyName, char* key1, char*k2); + jobject StopRegionServer(jobject cluster, int idx); + + // moves region to server + void MoveRegion(char *region, char *server); + // returns the Configuration instance for the cluster + jobject GetConf(jobject cluster); + // returns the value for config key retrieved from cluster + const char* GetConf(jobject cluster, char* key); + // Does Put into table for family fam, qualifier col with value + jobject TablePut(char *table, char *row, char *fam, char *col, char *value); + + private: + JNIEnv* env; + jclass testingUtilClass; + jclass tableNameClass; + jclass putClass; + jclass confClass; + jmethodID stopRSMid; + jmethodID getConfMid; + jmethodID setConfMid; + jmethodID tblNameValueOfMid; + jmethodID createTableMid; + jmethodID createTableWithSplitMid; + jmethodID putMid; + jmethodID putCtor; + jmethodID addColMid; + jmethodID createConnMid; + jmethodID getConnMid; + jmethodID getTableMid; + jmethodID confGetMid; + jmethodID getAdminMid; + jmethodID moveMid; + jmethodID strCtorMid; + pthread_mutex_t count_mutex; + jobject htu; + jobject cluster; + JavaVM *jvm; + JNIEnv* CreateVM(JavaVM **jvm); + void WriteConf(jobject conf, const char *filepath); + void setup(); + jobject getHTU(); + JNIEnv* getEnv(); + jbyteArray str_to_byte_array(char *str); + jobject getAdmin(); + }; +} /*namespace hbase*/