diff --git a/hbase-native-client/Makefile b/hbase-native-client/Makefile index 84ae556..fba7e09 100644 --- a/hbase-native-client/Makefile +++ b/hbase-native-client/Makefile @@ -22,7 +22,7 @@ LD:=g++ DEBUG_PATH = build/debug RELEASE_PATH = build/release PROTO_SRC_DIR = build/if -MODULES = connection core serde test-util utils security +MODULES = common connection core serde test-util utils security SRC_DIR = $(MODULES) DEBUG_BUILD_DIR = $(addprefix $(DEBUG_PATH)/,$(MODULES)) RELEASE_BUILD_DIR = $(addprefix $(RELEASE_PATH)/,$(MODULES)) diff --git a/hbase-native-client/core/BUCK b/hbase-native-client/core/BUCK index e541d8f..fed7f1d 100644 --- a/hbase-native-client/core/BUCK +++ b/hbase-native-client/core/BUCK @@ -33,8 +33,6 @@ cxx_library( "meta-utils.h", "get.h", "time_range.h", - "configuration.h", - "hbase_configuration_loader.h", "scan.h", "result.h", "request_converter.h", @@ -49,8 +47,6 @@ cxx_library( "meta-utils.cc", "get.cc", "time_range.cc", - "configuration.cc", - "hbase_configuration_loader.cc", "scan.cc", "result.cc", "request_converter.cc", @@ -58,6 +54,7 @@ cxx_library( "table.cc", ], deps=[ + "//common:common", "//connection:connection", "//if:if", "//serde:serde", @@ -101,16 +98,6 @@ cxx_test( deps=[":core",], run_test_separately=True,) cxx_test( - name="configuration-test", - srcs=["configuration-test.cc",], - deps=[":core",], - run_test_separately=True,) -cxx_test( - name="hbase_configuration-test", - srcs=["hbase_configuration-test.cc",], - deps=[":core",], - run_test_separately=True,) -cxx_test( name="scan-test", srcs=["scan-test.cc",], deps=[":core",], diff --git a/hbase-native-client/core/client-test.cc b/hbase-native-client/core/client-test.cc index 0a45fff..c6171ee 100644 --- a/hbase-native-client/core/client-test.cc +++ b/hbase-native-client/core/client-test.cc @@ -19,9 +19,9 @@ #include "core/client.h" #include -#include "core/configuration.h" +#include "common/configuration.h" +#include "common/hbase_configuration_loader.h" #include "core/get.h" -#include "core/hbase_configuration_loader.h" #include "core/result.h" #include "core/table.h" #include "serde/table-name.h" diff --git a/hbase-native-client/core/client.h b/hbase-native-client/core/client.h index a96d6f3..aadb6e0 100644 --- a/hbase-native-client/core/client.h +++ b/hbase-native-client/core/client.h @@ -28,9 +28,9 @@ #include #include "connection/rpc-client.h" -#include "core/configuration.h" +#include "common/configuration.h" #include "core/connection-configuration.h" -#include "core/hbase_configuration_loader.h" +#include "common/hbase_configuration_loader.h" #include "core/keyvalue-codec.h" #include "core/location-cache.h" #include "core/table.h" diff --git a/hbase-native-client/core/configuration-test.cc b/hbase-native-client/core/configuration-test.cc deleted file mode 100644 index 192ed46..0000000 --- a/hbase-native-client/core/configuration-test.cc +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -#include "core/configuration.h" -#include - -using hbase::Configuration; - -TEST(Configuration, SetGet) { - Configuration conf; - - EXPECT_EQ(conf.Get("foo", "default"), "default"); - conf.Set("foo", "bar"); - EXPECT_EQ(conf.Get("foo", "default"), "bar"); -} - -TEST(Configuration, SetGetInt) { - Configuration conf; - - EXPECT_EQ(conf.GetInt("foo", 0), 0); - conf.SetInt("foo", 42); - EXPECT_EQ(conf.GetInt("foo", 0), 42); -} - -TEST(Configuration, SetGetLong) { - Configuration conf; - - EXPECT_EQ(conf.GetLong("foo", 0), 0); - conf.SetLong("foo", 42); - EXPECT_EQ(conf.GetLong("foo", 0), 42); -} - -TEST(Configuration, SetGetDouble) { - Configuration conf; - - EXPECT_EQ(conf.GetDouble("foo", 0), 0); - conf.SetDouble("foo", 42.0); - EXPECT_EQ(conf.GetDouble("foo", 0), 42.0); -} - -TEST(Configuration, SetGetBool) { - Configuration conf; - - EXPECT_EQ(conf.GetBool("foo", false), false); - conf.SetInt("foo", true); - EXPECT_EQ(conf.GetInt("foo", false), true); -} diff --git a/hbase-native-client/core/configuration.cc b/hbase-native-client/core/configuration.cc deleted file mode 100644 index afca122..0000000 --- a/hbase-native-client/core/configuration.cc +++ /dev/null @@ -1,240 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -#include "core/configuration.h" - -#include -#include -#include - -#include -#include - -namespace hbase { - -Configuration::Configuration() : hb_property_() {} - -Configuration::Configuration(ConfigMap &config_map) : hb_property_(std::move(config_map)) {} - -Configuration::~Configuration() {} - -size_t Configuration::IsSubVariable(const std::string &expr, std::string &sub_variable) const { - size_t start_pos = expr.find("${"); - if (std::string::npos != start_pos) { - size_t pos_next = expr.find("}", start_pos + 1); - if (std::string::npos != pos_next) { - sub_variable = expr.substr(start_pos + 2, pos_next - (start_pos + 2)); - } - } - return start_pos; -} - -std::string Configuration::SubstituteVars(const std::string &expr) const { - if (0 == expr.size()) return expr; - - std::string eval(expr); - std::string value_to_be_replaced(""); - std::string var(""); - for (int i = 0; i < kMaxSubsts; i++) { - var = ""; - size_t start_pos = IsSubVariable(eval, var); - if (start_pos != std::string::npos) { - // We are blindly checking for environment property at first. - // If we don't get any value from GetEnv, check in hbase-site.xml. - value_to_be_replaced = GetEnv(var).value_or(GetProperty(var).value_or("")); - - // we haven't found any value yet so we are returning eval - if (0 == value_to_be_replaced.size()) { - return eval; - } - - // return original expression if there is a loop - if (value_to_be_replaced == expr) { - return expr; - } - - eval.replace(start_pos, var.size() + 3, value_to_be_replaced); - - } else { - // No further expansion required. - return eval; - } - } - // We reached here if the loop is exhausted - // If MAX_SUBSTS is exhausted, check if more variable substitution is reqd. - // If any-more substitutions are reqd, throw an error. - var = ""; - if (IsSubVariable(eval, var) != std::string::npos) { - throw std::runtime_error("Variable substitution depth too large: " + - std::to_string(kMaxSubsts) + " " + expr); - } else { - return eval; - } -} - -optional Configuration::GetEnv(const std::string &key) const { - char buf[2048]; - - if ("user.name" == key) { -#ifdef HAVE_GETLOGIN - return std::experimental::make_optional(getlogin()); -#else - DLOG(WARNING) << "Client user.name not implemented"; - return optional(); -#endif - } - - if ("user.dir" == key) { -#ifdef HAVE_GETCWD - if (getcwd(buf, sizeof(buf))) { - return std::experimental::make_optional(buf); - } else { - return optional(); - } -#else - DLOG(WARNING) << "Client user.dir not implemented"; - return optional(); -#endif - } - - if ("user.home" == key) { -#if defined(HAVE_GETUID) && defined(HAVE_GETPWUID_R) - uid = getuid(); - if (!getpwuid_r(uid, &pw, buf, sizeof(buf), &pwp)) { - return std::experimental::make_optional(buf); - } else { - return optional(); - } -#else - DLOG(WARNING) << "Client user.home not implemented"; - return optional(); -#endif - } - return optional(); -} - -optional Configuration::GetProperty(const std::string &key) const { - auto found = hb_property_.find(key); - if (found != hb_property_.end()) { - return std::experimental::make_optional(found->second.value); - } else { - return optional(); - } -} - -optional Configuration::Get(const std::string &key) const { - optional raw = GetProperty(key); - if (raw) { - return std::experimental::make_optional(SubstituteVars(*raw)); - } else { - return optional(); - } -} - -std::string Configuration::Get(const std::string &key, const std::string &default_value) const { - return Get(key).value_or(default_value); -} - -optional Configuration::GetInt(const std::string &key) const { - optional raw = Get(key); - if (raw) { - try { - return std::experimental::make_optional(boost::lexical_cast(*raw)); - } catch (const boost::bad_lexical_cast &blex) { - throw std::runtime_error(blex.what()); - } - } - return optional(); -} - -int32_t Configuration::GetInt(const std::string &key, int32_t default_value) const { - return GetInt(key).value_or(default_value); -} - -optional Configuration::GetLong(const std::string &key) const { - optional raw = Get(key); - if (raw) { - try { - return std::experimental::make_optional(boost::lexical_cast(*raw)); - } catch (const boost::bad_lexical_cast &blex) { - throw std::runtime_error(blex.what()); - } - } - return optional(); -} - -int64_t Configuration::GetLong(const std::string &key, int64_t default_value) const { - return GetLong(key).value_or(default_value); -} - -optional Configuration::GetDouble(const std::string &key) const { - optional raw = Get(key); - if (raw) { - try { - return std::experimental::make_optional(boost::lexical_cast(*raw)); - } catch (const boost::bad_lexical_cast &blex) { - throw std::runtime_error(blex.what()); - } - } - return optional(); -} - -double Configuration::GetDouble(const std::string &key, double default_value) const { - return GetDouble(key).value_or(default_value); -} - -optional Configuration::GetBool(const std::string &key) const { - optional raw = Get(key); - if (raw) { - if (!strcasecmp((*raw).c_str(), "true")) { - return std::experimental::make_optional(true); - } else if (!strcasecmp((*raw).c_str(), "false")) { - return std::experimental::make_optional(false); - } else { - throw std::runtime_error("Unexpected value found while conversion to bool."); - } - } - return optional(); -} - -bool Configuration::GetBool(const std::string &key, bool default_value) const { - return GetBool(key).value_or(default_value); -} - -void Configuration::Set(const std::string &key, const std::string &value) { - hb_property_[key] = value; -} - -void Configuration::SetInt(const std::string &key, int32_t value) { - Set(key, boost::lexical_cast(value)); -} - -void Configuration::SetLong(const std::string &key, int64_t value) { - Set(key, boost::lexical_cast(value)); -} - -void Configuration::SetDouble(const std::string &key, double value) { - Set(key, boost::lexical_cast(value)); -} - -void Configuration::SetBool(const std::string &key, bool value) { - Set(key, boost::lexical_cast(value)); -} - -} /* namespace hbase */ diff --git a/hbase-native-client/core/configuration.h b/hbase-native-client/core/configuration.h deleted file mode 100644 index d70941c..0000000 --- a/hbase-native-client/core/configuration.h +++ /dev/null @@ -1,232 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -/* Some pieces of code have been added from HDFS-8707 */ -#pragma once - -#include -#include -#include - -#include - -namespace hbase { - -template -using optional = std::experimental::optional; - -class Configuration { - public: - /** - * @brief Create Configuration object using an empty map. This - * object WILL NOT be initialized from reading XML configuration (hbase-site.xml, etc). - * Use HBaseConfigurationLoader to initialize the configuration from - * hbase-default.xml and hbase-site.xml files. - */ - Configuration(); - - ~Configuration(); - - /** - * @brief Returns value identified by key in ConfigMap else default value if - * property is absent. - * @param key Property whose value is to be fetched. SubstituteVars will be - * called for any variable expansion. - */ - std::string Get(const std::string &key, const std::string &default_value) const; - - /** - * @brief Returns int32_t identified by key in ConfigMap else default value if - * property is absent. - * @param key Property whose value is to be fetched. Internally Get(key) will - * be called. - * @throws std::runtime_error if conversion to int32_t fails - */ - int32_t GetInt(const std::string &key, int32_t default_value) const; - - /** - * @brief Returns int64_t identified by key in ConfigMap else default value if - * property is absent. - * @param key Property whose value is to be fetched. Internally Get(key) will - * be called and - * @throws std::runtime_error if conversion to int64_t fails - */ - int64_t GetLong(const std::string &key, int64_t default_value) const; - - /** - * @brief Returns double identified by key in ConfigMap else default value if - * property is absent. - * @param key Property whose value is to be fetched. Internally Get(key) will - * be called. - * @throws std::runtime_error if conversion to double fails - */ - double GetDouble(const std::string &key, double default_value) const; - - /** - * @brief Returns bool identified by key in ConfigMap else default value if - * property is absent. - * @param key Property whose value is to be fetched. Internally Get(key) will - * be called. - * @throws std::runtime_error if conversion to bool fails - */ - bool GetBool(const std::string &key, bool default_value) const; - - /** - * @brief This method sets the given key to the value. - * @param key property name - * @param value property value - */ - void Set(const std::string &key, const std::string &value); - - /** - * @brief This method sets the given key to the value. - * @param key property name - * @param value property value - */ - void SetInt(const std::string &key, int32_t value); - - /** - * @brief This method sets the given key to the value. - * @param key property name - * @param value property value - */ - void SetLong(const std::string &key, int64_t value); - - /** - * @brief This method sets the given key to the value. - * @param key property name - * @param value property value - */ - void SetDouble(const std::string &key, double value); - - /** - * @brief This method sets the given key to the value. - * @param key property name - * @param value property value - */ - void SetBool(const std::string &key, bool value); - - private: - friend class HBaseConfigurationLoader; - - /* Transparent data holder for property values */ - /* Same as Jira HDFS-8707 */ - struct ConfigData { - std::string value; - bool final; - ConfigData() : final(false) {} - explicit ConfigData(const std::string &value) : value(value), final(false) {} - void operator=(const std::string &new_value) { - value = new_value; - final = false; - } - }; - - /** - * @brief Map which hold configuration properties. - */ - using ConfigMap = std::map; - - /** - * @brief Create Configuration object using config_map; - * @param config_map - Map consisting of properties. - */ - explicit Configuration(ConfigMap &config_map); - - // Property map filled all the properties loaded by ConfigurationLoader - ConfigMap hb_property_; - - // Variable expansion depth - const int kMaxSubsts = 20; - - /** - * @brief returns value for a property identified by key in ConfigMap. - * SubstituteVars will be called for any variable expansion. - * @param key Key whose value is to be fetched. - */ - optional Get(const std::string &key) const; - - /** - * @brief returns optional int32_t value identified by the key in ConfigMap. - * Get(key) is called to get the string value which is then converted to - * int32_t using boost::lexical_cast. - * @param key Key whose value is to be fetched. - * @throws std::runtime_error if conversion to int32_t fails - */ - optional GetInt(const std::string &key) const; - - /** - * @brief returns optional int64_t value identified by the key in ConfigMap. - * Get(key) is called internally to get the string value which is then - * converted to int64_t using boost::lexical_cast. - * @param key Key whose value is to be fetched. - * @throws std::runtime_error if conversion to int64_t fails - */ - optional GetLong(const std::string &key) const; - - /** - * @brief returns optional double value identified by the key in ConfigMap. - * Get(key) is called to get the string value which is then converted to - * double using boost::lexical_cast. - * @param key Key whose value is to be fetched. - * @throws std::runtime_error if conversion to double fails - */ - optional GetDouble(const std::string &key) const; - - /** - * @brief returns optional bool for a property identified by key in ConfigMap. - * Get(key) is called to get the string value which is then converted to bool - * by checking the validity. - * @param key Key whose value is to be fetched. Get(key) is called to get the - * string value which is then converted to bool. - * @throws std::runtime_error if conversion to bool fails - */ - optional GetBool(const std::string &key) const; - - /** - * @brief This method will perform any variable expansion if present. - * @param expression expression string to perform substitutions on. - * @throws std::runtime_error if MAX_SUBSTS is exhausted and any more - * substitutions are reqd to be performed. - */ - std::string SubstituteVars(const std::string &expr) const; - - /** - * @brief This method will check if variable expansion has to be performed or - * not. Expression will be checked for presence of "${" and "}" to perform - * variable expansion. - * @param expr Expression on which will be checked for validity. - * @param sub_variable Extracted variable from expr which will be checked - * against environment value or ConfigMap values. - */ - size_t IsSubVariable(const std::string &expr, std::string &sub_variable) const; - - /** - * @brief This method will fetch value for key from environment if present. - * @param key key to be fetched from environment. - */ - optional GetEnv(const std::string &key) const; - - /** - * @brief This method will fetch value for key from ConfigMap if present. - * @param key key to be fetched from environment. - */ - optional GetProperty(const std::string &key) const; -}; -} /* namespace hbase */ diff --git a/hbase-native-client/core/connection-configuration.h b/hbase-native-client/core/connection-configuration.h index e1e9f87..48d0ec4 100644 --- a/hbase-native-client/core/connection-configuration.h +++ b/hbase-native-client/core/connection-configuration.h @@ -23,7 +23,7 @@ #include #include -#include "core/configuration.h" +#include "common/configuration.h" using std::chrono::nanoseconds; using std::chrono::milliseconds; diff --git a/hbase-native-client/core/filter-test.cc b/hbase-native-client/core/filter-test.cc index ff683b6..9728d1b 100644 --- a/hbase-native-client/core/filter-test.cc +++ b/hbase-native-client/core/filter-test.cc @@ -18,8 +18,8 @@ */ #include +#include "common/configuration.h" #include "core/client.h" -#include "core/configuration.h" #include "core/get.h" #include "core/result.h" #include "core/table.h" diff --git a/hbase-native-client/core/hbase_configuration-test.cc b/hbase-native-client/core/hbase_configuration-test.cc deleted file mode 100644 index 07a6358..0000000 --- a/hbase-native-client/core/hbase_configuration-test.cc +++ /dev/null @@ -1,374 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -#include -#include - -#include -#include -#include -#include "core/configuration.h" -#include "core/hbase_configuration_loader.h" - -using namespace hbase; - -const std::string kDefHBaseConfPath("./build/test-data/hbase-configuration-test/conf/"); -const std::string kHBaseConfPath("./build/test-data/hbase-configuration-test/custom-conf/"); - -const std::string kHBaseDefaultXml("hbase-default.xml"); -const std::string kHBaseSiteXml("hbase-site.xml"); - -const std::string kHBaseDefaultXmlData( - "\n\n\n\n\n\nhbase.rootdir\n/root/hbase-docker/apps/hbase/data\ntrue\n\n\n\nhbase.zookeeper.property.datadir\nThis value will be " - "overwritten\nfalse\n\n\n\ndefault-prop\ndefault-value\n\n\n"); -const std::string kHBaseSiteXmlData( - "\n\n\n\n\n\nhbase.rootdir\nThis value will not be be " - "overwritten\n\n\n\nhbase.zookeeper.property.datadir\n/root/hbase-docker/zookeeper\n\n\n\nhbase-client.user.name\n${user.name}\n\n\n\nhbase-client.user.dir\n${user.dir}\n\n\n\nhbase-client.user.home\n${user.home}\n\n\n\nselfRef\n${selfRef}\n\n\n\nfoo.substs\n${bar},${bar},${bar},${bar},${bar},${bar},${bar},${bar},${" - "bar},${bar},\n\n\n\nfoo.substs.exception\n${bar},${bar},${bar},${bar},${bar},${bar},${bar},${bar},${" - "bar},${bar},${bar},${bar},${bar},${bar},${bar},${bar},${bar},${bar},${bar}" - ",${bar},${bar}\n\n\n\nbar\nbar-value\n\n\n\ncustom-prop\ncustom-value\n\n\n\nint\n16000\n\n\n\nint.largevalue\n2147483646\n\n\n\nint.exception\n2147483648\n\n\n\nlong\n2147483850\n\n\n\nlong.largevalue\n9223372036854775807\n\n\n\nlong.exception\n9223372036854775810\n\n\n\ndouble\n17.9769e+100\n\n\n\ndouble.largevalue\n170.769e+200\n\n\n\ndouble.exception\n1.79769e+310\n\n\n\nbool.true\ntrue\n\n\n\nbool.false\nfalse\n\n\n\nbool.exception\nunknown " - "bool\n\n\n"); - -void WriteDataToFile(const std::string &file, const std::string &xml_data) { - std::ofstream hbase_conf; - hbase_conf.open(file.c_str()); - hbase_conf << xml_data; - hbase_conf.close(); -} - -void CreateHBaseConf(const std::string &dir, const std::string &file, const std::string xml_data) { - // Directory will be created if not present - if (!boost::filesystem::exists(dir)) { - boost::filesystem::create_directories(dir); - } - // Remove temp file always - boost::filesystem::remove((dir + file).c_str()); - WriteDataToFile((dir + file), xml_data); -} - -void CreateHBaseConfWithEnv() { - CreateHBaseConf(kDefHBaseConfPath, kHBaseDefaultXml, kHBaseDefaultXmlData); - CreateHBaseConf(kDefHBaseConfPath, kHBaseSiteXml, kHBaseSiteXmlData); - setenv("HBASE_CONF", kDefHBaseConfPath.c_str(), 1); -} - -/* - * Config will be loaded from $HBASE_CONF. We set it @ kDefHBaseConfPath - * Config values will be loaded from hbase-default.xml and hbase-site.xml - * present in the above path. - */ -TEST(Configuration, LoadConfFromDefaultLocation) { - // Remove already configured env if present. - unsetenv("HBASE_CONF"); - CreateHBaseConf(kDefHBaseConfPath, kHBaseDefaultXml, kHBaseDefaultXmlData); - CreateHBaseConf(kDefHBaseConfPath, kHBaseSiteXml, kHBaseSiteXmlData); - setenv("HBASE_CONF", kDefHBaseConfPath.c_str(), 0); - - HBaseConfigurationLoader loader; - hbase::optional conf = loader.LoadDefaultResources(); - ASSERT_TRUE(conf) << "No configuration object present."; - EXPECT_STREQ((*conf).Get("custom-prop", "Set this value").c_str(), "custom-value"); - EXPECT_STREQ((*conf).Get("default-prop", "Set this value").c_str(), "default-value"); -} - -/* - * Config will be loaded from hbase-site.xml defined at - * kHBaseConfPath - */ -TEST(Configuration, LoadConfFromCustomLocation) { - // Remove already configured env if present. - unsetenv("HBASE_CONF"); - CreateHBaseConf(kHBaseConfPath, kHBaseSiteXml, kHBaseSiteXmlData); - - HBaseConfigurationLoader loader; - std::vector resources{kHBaseSiteXml}; - hbase::optional conf = loader.LoadResources(kHBaseConfPath, resources); - ASSERT_TRUE(conf) << "No configuration object present."; - EXPECT_STREQ((*conf).Get("custom-prop", "").c_str(), "custom-value"); - EXPECT_STRNE((*conf).Get("custom-prop", "").c_str(), "some-value"); -} - -/* - * Config will be loaded from hbase-defualt.xml and hbase-site.xml @ - * kDefHBaseConfPath and kHBaseConfPath respectively. - */ -TEST(Configuration, LoadConfFromMultipleLocatons) { - // Remove already configured env if present. - unsetenv("HBASE_CONF"); - CreateHBaseConf(kDefHBaseConfPath, kHBaseDefaultXml, kHBaseDefaultXmlData); - CreateHBaseConf(kDefHBaseConfPath, kHBaseSiteXml, kHBaseSiteXmlData); - CreateHBaseConf(kHBaseConfPath, kHBaseDefaultXml, kHBaseDefaultXmlData); - CreateHBaseConf(kHBaseConfPath, kHBaseSiteXml, kHBaseSiteXmlData); - - HBaseConfigurationLoader loader; - std::string conf_paths = kDefHBaseConfPath + ":" + kHBaseConfPath; - std::vector resources{kHBaseDefaultXml, kHBaseSiteXml}; - hbase::optional conf = loader.LoadResources(conf_paths, resources); - ASSERT_TRUE(conf) << "No configuration object present."; - EXPECT_STREQ((*conf).Get("default-prop", "From hbase-default.xml").c_str(), "default-value"); - EXPECT_STREQ((*conf).Get("custom-prop", "").c_str(), "custom-value"); - EXPECT_STRNE((*conf).Get("custom-prop", "").c_str(), "some-value"); -} - -/* - * Config will be loaded from hbase-defualt.xml and hbase-site.xml @ - * $HBASE_CONF. - * We set HBASE_CONF to kDefHBaseConfPath - * Below tests load the conf files in the same way unless specified. - */ -TEST(Configuration, DefaultValues) { - // Remove already configured env if present. - unsetenv("HBASE_CONF"); - CreateHBaseConfWithEnv(); - - HBaseConfigurationLoader loader; - hbase::optional conf = loader.LoadDefaultResources(); - ASSERT_TRUE(conf) << "No configuration object present."; - EXPECT_STREQ((*conf).Get("default-prop", "Set this value.").c_str(), "default-value"); - EXPECT_STREQ((*conf).Get("custom-prop", "Set this value.").c_str(), "custom-value"); -} - -TEST(Configuration, FinalValues) { - // Remove already configured env if present. - unsetenv("HBASE_CONF"); - CreateHBaseConfWithEnv(); - - HBaseConfigurationLoader loader; - hbase::optional conf = loader.LoadDefaultResources(); - ASSERT_TRUE(conf) << "No configuration object present."; - EXPECT_STREQ((*conf).Get("hbase.rootdir", "").c_str(), "/root/hbase-docker/apps/hbase/data"); - EXPECT_STREQ((*conf).Get("hbase.zookeeper.property.datadir", "").c_str(), - "/root/hbase-docker/zookeeper"); - EXPECT_STRNE((*conf).Get("hbase.rootdir", "").c_str(), "This value will not be be overwritten"); - EXPECT_STRNE((*conf).Get("hbase.zookeeper.property.datadir", "").c_str(), - "This value will be overwritten"); -} - -/* - * Config will be loaded from HBASE_CONF which we set in - * CreateHBaseConfWithEnv(). - * Config values will be loaded from hbase-default.xml and hbase-site.xml in the - * above path. - */ -TEST(Configuration, EnvVars) { - // Remove already configured env if present. - unsetenv("HBASE_CONF"); - CreateHBaseConfWithEnv(); - - HBaseConfigurationLoader loader; - hbase::optional conf = loader.LoadDefaultResources(); - ASSERT_TRUE(conf) << "No configuration object present."; - EXPECT_STREQ((*conf).Get("hbase-client.user.name", "").c_str(), "${user.name}"); - EXPECT_STRNE((*conf).Get("hbase-client.user.name", "root").c_str(), "test-user"); -} - -TEST(Configuration, SelfRef) { - // Remove already configured env if present. - unsetenv("HBASE_CONF"); - CreateHBaseConfWithEnv(); - - HBaseConfigurationLoader loader; - hbase::optional conf = loader.LoadDefaultResources(); - ASSERT_TRUE(conf) << "No configuration object present."; - EXPECT_STREQ((*conf).Get("selfRef", "${selfRef}").c_str(), "${selfRef}"); -} - -TEST(Configuration, VarExpansion) { - // Remove already configured env if present. - unsetenv("HBASE_CONF"); - CreateHBaseConfWithEnv(); - - HBaseConfigurationLoader loader; - hbase::optional conf = loader.LoadDefaultResources(); - ASSERT_TRUE(conf) << "No configuration object present."; - EXPECT_STREQ((*conf).Get("foo.substs", "foo-value").c_str(), - "bar-value,bar-value,bar-value,bar-value,bar-value,bar-value," - "bar-value,bar-value,bar-value,bar-value,"); - EXPECT_STRNE((*conf).Get("foo.substs", "foo-value").c_str(), "bar-value"); -} - -TEST(Configuration, VarExpansionException) { - // Remove already configured env if present. - unsetenv("HBASE_CONF"); - CreateHBaseConfWithEnv(); - - HBaseConfigurationLoader loader; - hbase::optional conf = loader.LoadDefaultResources(); - ASSERT_TRUE(conf) << "No configuration object present."; - ASSERT_THROW((*conf).Get("foo.substs.exception", "foo-value").c_str(), std::runtime_error); -} - -TEST(Configuration, GetInt) { - // Remove already configured env if present. - unsetenv("HBASE_CONF"); - CreateHBaseConfWithEnv(); - - HBaseConfigurationLoader loader; - hbase::optional conf = loader.LoadDefaultResources(); - ASSERT_TRUE(conf) << "No configuration object present."; - EXPECT_EQ(16000, (*conf).GetInt("int", 0)); - EXPECT_EQ(2147483646, (*conf).GetInt("int.largevalue", 0)); -} - -TEST(Configuration, GetLong) { - // Remove already configured env if present. - unsetenv("HBASE_CONF"); - CreateHBaseConfWithEnv(); - - HBaseConfigurationLoader loader; - hbase::optional conf = loader.LoadDefaultResources(); - ASSERT_TRUE(conf) << "No configuration object present."; - EXPECT_EQ(2147483850, (*conf).GetLong("long", 0)); - EXPECT_EQ(9223372036854775807, (*conf).GetLong("long.largevalue", 0)); -} - -TEST(Configuration, GetDouble) { - // Remove already configured env if present. - unsetenv("HBASE_CONF"); - CreateHBaseConfWithEnv(); - - HBaseConfigurationLoader loader; - hbase::optional conf = loader.LoadDefaultResources(); - ASSERT_TRUE(conf) << "No configuration object present."; - EXPECT_DOUBLE_EQ(17.9769e+100, (*conf).GetDouble("double", 0.0)); - EXPECT_DOUBLE_EQ(170.769e+200, (*conf).GetDouble("double.largevalue", 0.0)); -} - -TEST(Configuration, GetBool) { - // Remove already configured env if present. - unsetenv("HBASE_CONF"); - CreateHBaseConfWithEnv(); - - HBaseConfigurationLoader loader; - hbase::optional conf = loader.LoadDefaultResources(); - ASSERT_TRUE(conf) << "No configuration object present."; - EXPECT_EQ(true, (*conf).GetBool("bool.true", true)); - EXPECT_EQ(false, (*conf).GetBool("bool.false", false)); -} - -TEST(Configuration, GetIntException) { - // Remove already configured env if present. - unsetenv("HBASE_CONF"); - CreateHBaseConfWithEnv(); - - HBaseConfigurationLoader loader; - hbase::optional conf = loader.LoadDefaultResources(); - ASSERT_TRUE(conf) << "No configuration object present."; - ASSERT_THROW((*conf).GetInt("int.exception", 0), std::runtime_error); -} - -TEST(Configuration, GetLongException) { - // Remove already configured env if present. - unsetenv("HBASE_CONF"); - CreateHBaseConfWithEnv(); - - HBaseConfigurationLoader loader; - hbase::optional conf = loader.LoadDefaultResources(); - ASSERT_TRUE(conf) << "No configuration object present."; - ASSERT_THROW((*conf).GetLong("long.exception", 0), std::runtime_error); -} - -TEST(Configuration, GetDoubleException) { - // Remove already configured env if present. - unsetenv("HBASE_CONF"); - CreateHBaseConfWithEnv(); - - HBaseConfigurationLoader loader; - hbase::optional conf = loader.LoadDefaultResources(); - ASSERT_TRUE(conf) << "No configuration object present."; - ASSERT_THROW((*conf).GetDouble("double.exception", 0), std::runtime_error); -} - -TEST(Configuration, GetBoolException) { - // Remove already configured env if present. - unsetenv("HBASE_CONF"); - CreateHBaseConfWithEnv(); - - HBaseConfigurationLoader loader; - hbase::optional conf = loader.LoadDefaultResources(); - ASSERT_TRUE(conf) << "No configuration object present."; - ASSERT_THROW((*conf).GetBool("bool.exception", false), std::runtime_error); -} diff --git a/hbase-native-client/core/hbase_configuration_loader.cc b/hbase-native-client/core/hbase_configuration_loader.cc deleted file mode 100644 index 38a617b..0000000 --- a/hbase-native-client/core/hbase_configuration_loader.cc +++ /dev/null @@ -1,208 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -#include "core/hbase_configuration_loader.h" - -#include -#include -#include -#include - -namespace hbase { - -bool is_valid_bool(const std::string &raw) { - if (raw.empty()) { - return false; - } - - if (!strcasecmp(raw.c_str(), "true")) { - return true; - } - if (!strcasecmp(raw.c_str(), "false")) { - return true; - } - return false; -} - -bool str_to_bool(const std::string &raw) { - if (!strcasecmp(raw.c_str(), "true")) { - return true; - } - return false; -} - -HBaseConfigurationLoader::HBaseConfigurationLoader() {} - -HBaseConfigurationLoader::~HBaseConfigurationLoader() {} - -void HBaseConfigurationLoader::SetDefaultSearchPath() { - /* - * Try (in order, taking the first valid one): - * $HBASE_CONF_DIR - * /etc/hbase/conf - * - */ - const char *hadoop_conf_dir_env = getenv("HBASE_CONF"); - if (hadoop_conf_dir_env) { - AddToSearchPath(hadoop_conf_dir_env); - } else { - AddToSearchPath(kHBaseDefauktConfPath); - } -} - -void HBaseConfigurationLoader::ClearSearchPath() { search_paths_.clear(); } - -void HBaseConfigurationLoader::SetSearchPath(const std::string &search_path) { - search_paths_.clear(); - - std::vector paths; - std::string::size_type start = 0; - std::string::size_type end = search_path.find(kSearchPathSeparator); - - while (end != std::string::npos) { - paths.push_back(search_path.substr(start, end - start)); - start = ++end; - end = search_path.find(kSearchPathSeparator, start); - } - paths.push_back(search_path.substr(start, search_path.length())); - - for (auto path : paths) { - AddToSearchPath(path); - } -} - -void HBaseConfigurationLoader::AddToSearchPath(const std::string &search_path) { - if (search_path.empty()) return; - - std::string path_to_add(search_path); - if (search_path.back() != kFileSeparator) { - path_to_add += kFileSeparator; - } - if (std::find(search_paths_.begin(), search_paths_.end(), path_to_add) == search_paths_.end()) - search_paths_.push_back(path_to_add); -} - -void HBaseConfigurationLoader::AddDefaultResources() { - resources_.push_back(kHBaseDefaultXml); - resources_.push_back(kHBaseSiteXml); -} - -void HBaseConfigurationLoader::AddResources(const std::string &filename) { - if (std::find(resources_.begin(), resources_.end(), filename) == resources_.end()) - resources_.push_back(filename); -} - -optional HBaseConfigurationLoader::LoadDefaultResources() { - SetDefaultSearchPath(); - AddDefaultResources(); - ConfigMap conf_property; - bool success = false; - for (auto dir : search_paths_) { - for (auto file : resources_) { - std::string config_file = dir + file; - std::ifstream stream(config_file); - if (stream.is_open()) { - success |= LoadProperties(config_file, conf_property); - } else { - DLOG(WARNING) << "Unable to open file[" << config_file << "]"; - } - } - } - if (success) { - return std::experimental::make_optional(Configuration(conf_property)); - } else { - return optional(); - } -} - -optional HBaseConfigurationLoader::LoadResources( - const std::string &search_path, const std::vector &resources) { - SetSearchPath(search_path); - for (const auto &resource : resources) AddResources(resource); - ConfigMap conf_property; - bool success = false; - for (auto dir : search_paths_) { - for (auto file : resources_) { - std::string config_file = dir + file; - std::ifstream stream(config_file); - if (stream.is_open()) { - success |= LoadProperties(config_file, conf_property); - } else { - DLOG(WARNING) << "Unable to open file[" << config_file << "]"; - } - } - } - if (success) { - return std::experimental::make_optional(Configuration(conf_property)); - } else { - return optional(); - } -} - -bool HBaseConfigurationLoader::LoadProperties(const std::string &file, ConfigMap &property_map) { - // Create empty property tree object - using boost::property_tree::ptree; - ptree pt; - try { - // Load XML file and put contents in a property tree. - // If read fails, throw exception. - read_xml(file, pt); - - // If configuration key is not found exception is thrown - std::string configuration = pt.get("configuration"); - - // Iterate over configuration section. - // Store all found properties in ConfigMap - BOOST_FOREACH (ptree::value_type &v, pt.get_child("configuration")) { - if ("property" == v.first) { - std::string name_node = v.second.get("name"); - std::string value_node = v.second.get("value"); - if ((name_node.size() > 0) && (value_node.size() > 0)) { - boost::optional final_node = v.second.get_optional("final"); - UpdateMapWithValue(property_map, name_node, value_node, final_node); - } - } - } - } catch (std::exception &ex) { - DLOG(WARNING) << "Exception in parsing file [" << file << "]:[" << ex.what() << "]"; - return false; - } - return true; -} - -bool HBaseConfigurationLoader::UpdateMapWithValue(ConfigMap &map, const std::string &key, - const std::string &value, - boost::optional final_text) { - auto map_value = map.find(key); - if (map_value != map.end() && map_value->second.final) { - return false; - } - - bool final_value = false; - if (nullptr != final_text.get_ptr()) { - if (is_valid_bool(final_text.get())) { - final_value = str_to_bool(final_text.get()); - } - } - - map[key].value = value; - map[key].final = final_value; - return true; -} -} /* namespace hbase */ diff --git a/hbase-native-client/core/hbase_configuration_loader.h b/hbase-native-client/core/hbase_configuration_loader.h deleted file mode 100644 index a1c1d3f..0000000 --- a/hbase-native-client/core/hbase_configuration_loader.h +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -#pragma once - -#include -#include -#include - -#include -#include - -#include "core/configuration.h" - -namespace hbase { - -template -using optional = std::experimental::optional; - -class HBaseConfigurationLoader { - public: - HBaseConfigurationLoader(); - ~HBaseConfigurationLoader(); - - /** - * @brief Creates a Configuration object based on default resources loaded - * from default search paths. Default search path will be either $HBASE_CONF - * is set or /etc/hbase/conf. Default resources are hbase-default.xml and - * hbase-site.xml.SetDefaultSearchPath() and AddDefaultResources() are used - * for the same. - * Values are loaded in from hbase-default.xml first and then from - * hbase-site.xml. - * Properties in hbase-site.xml will override the ones in hbase-default.xml - * unless marked as final - */ - optional LoadDefaultResources(); - - /* - * @brief Creates a Configuration object based on resources loaded from search - * paths. Search paths are defined in search_path. Values are loaded from - * resources and will be overridden unless marked as final - * @param search_path - ':' search paths to load resources. - * @param resources - list of resources used to load configuration properties. - */ - optional LoadResources(const std::string &search_path, - const std::vector &resources); - - private: - using ConfigMap = Configuration::ConfigMap; - const std::string kHBaseDefaultXml = "hbase-default.xml"; - const std::string kHBaseSiteXml = "hbase-site.xml"; - const std::string kHBaseDefauktConfPath = "/etc/hbase/conf"; - - // Adds FILE_SEPARATOR to the search path - const char kFileSeparator = '/'; - - // Separator using which multiple search paths can be defined. - const char kSearchPathSeparator = ':'; - - /** - * List of paths which will be looked up for loading properties. - */ - std::vector search_paths_; - - /** - * List of files which will be looked up in search_paths_ to load properties. - */ - std::vector resources_; - - /** - * @brief This method sets the search path to the default search path (i.e. - * "$HBASE_CONF" or "/etc/hbase/conf" if HBASE_CONF is absent) - */ - void SetDefaultSearchPath(); - - /** - * @brief Clears out the set search path(s) - */ - void ClearSearchPath(); - - /** - * @brief Sets the search path to ":"-delimited paths, clearing already - * defined values - * @param search_path Single path or ":"-delimited separated paths - */ - void SetSearchPath(const std::string &search_path); - - /** - * @brief Adds an element to the search path if not already present. - * @param search_path Path that will be added to load config values - */ - void AddToSearchPath(const std::string &search_path); - - /** - * @brief This method will add default resources i.e. hbase-default.xml and - * hbase-site.xml to the default search path. - */ - void AddDefaultResources(); - - /** - * @brief Adds resources to list for loading config values. - * @param filename to be added to resources. - */ - void AddResources(const std::string &filename); - - /** - * @brief Loads properties in file identified by file in a map identified by - * property_map - * @param file XML file which defines HBase configuration properties. - * @param property_map Property map representing HBase configuration as key - * value pairs. - * @throws Boost ptree exception if some parsing issue occurs. - */ - bool LoadProperties(const std::string &file, ConfigMap &property_map); - - /** - * @brief This method will create a map of the name and properties. - * @param map Property map to hold configuration properties. - * @param key value of name node. - * @param value value of value node. - * @param final_text value of final node true or false if present - */ - bool UpdateMapWithValue(ConfigMap &map, const std::string &key, const std::string &value, - boost::optional final_text); -}; - -} /* namespace hbase */ diff --git a/hbase-native-client/core/location-cache.h b/hbase-native-client/core/location-cache.h index 4d66e06..7d5714a 100644 --- a/hbase-native-client/core/location-cache.h +++ b/hbase-native-client/core/location-cache.h @@ -32,7 +32,7 @@ #include #include "connection/connection-pool.h" -#include "core/configuration.h" +#include "common/configuration.h" #include "core/meta-utils.h" #include "core/region-location.h" #include "serde/table-name.h" diff --git a/hbase-native-client/core/table.h b/hbase-native-client/core/table.h index 0e98cd2..aeb0077 100644 --- a/hbase-native-client/core/table.h +++ b/hbase-native-client/core/table.h @@ -25,7 +25,7 @@ #include "connection/rpc-client.h" #include "core/client.h" -#include "core/configuration.h" +#include "common/configuration.h" #include "core/get.h" #include "core/location-cache.h" #include "core/result.h" diff --git a/hbase-native-client/test-util/test-util.h b/hbase-native-client/test-util/test-util.h index cc35511..39be107 100644 --- a/hbase-native-client/test-util/test-util.h +++ b/hbase-native-client/test-util/test-util.h @@ -24,7 +24,7 @@ #include #include -#include "core/configuration.h" +#include "common/configuration.h" namespace hbase { /** diff --git a/hbase-native-client/common/BUCK b/hbase-native-client/common/BUCK new file mode 100644 index 0000000..0f771f5 --- /dev/null +++ b/hbase-native-client/common/BUCK @@ -0,0 +1,37 @@ +## +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is the library dealing with a single connection +# to a single server. +cxx_library( + name="common", + exported_headers=["hbase_configuration_loader.h","configuration.h"], + srcs=["hbase_configuration_loader.cc","configuration.cc"], + deps=["//third-party:folly"], + compiler_flags=['-Weffc++'], + linker_flags = [], + visibility=['//core/...', '//connection/...'],) +cxx_test( + name="configuration-test", + srcs=["configuration-test.cc",], + deps=[":common",], + run_test_separately=True,) +cxx_test( + name="hbase_configuration-test", + srcs=["hbase_configuration-test.cc",], + deps=[":common",], + run_test_separately=True,) diff --git a/hbase-native-client/common/configuration.cc b/hbase-native-client/common/configuration.cc new file mode 100644 index 0000000..a67e761 --- /dev/null +++ b/hbase-native-client/common/configuration.cc @@ -0,0 +1,240 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "common/configuration.h" + +#include +#include +#include + +#include +#include + +namespace hbase { + +Configuration::Configuration() : hb_property_() {} + +Configuration::Configuration(ConfigMap &config_map) : hb_property_(std::move(config_map)) {} + +Configuration::~Configuration() {} + +size_t Configuration::IsSubVariable(const std::string &expr, std::string &sub_variable) const { + size_t start_pos = expr.find("${"); + if (std::string::npos != start_pos) { + size_t pos_next = expr.find("}", start_pos + 1); + if (std::string::npos != pos_next) { + sub_variable = expr.substr(start_pos + 2, pos_next - (start_pos + 2)); + } + } + return start_pos; +} + +std::string Configuration::SubstituteVars(const std::string &expr) const { + if (0 == expr.size()) return expr; + + std::string eval(expr); + std::string value_to_be_replaced(""); + std::string var(""); + for (int i = 0; i < kMaxSubsts; i++) { + var = ""; + size_t start_pos = IsSubVariable(eval, var); + if (start_pos != std::string::npos) { + // We are blindly checking for environment property at first. + // If we don't get any value from GetEnv, check in hbase-site.xml. + value_to_be_replaced = GetEnv(var).value_or(GetProperty(var).value_or("")); + + // we haven't found any value yet so we are returning eval + if (0 == value_to_be_replaced.size()) { + return eval; + } + + // return original expression if there is a loop + if (value_to_be_replaced == expr) { + return expr; + } + + eval.replace(start_pos, var.size() + 3, value_to_be_replaced); + + } else { + // No further expansion required. + return eval; + } + } + // We reached here if the loop is exhausted + // If MAX_SUBSTS is exhausted, check if more variable substitution is reqd. + // If any-more substitutions are reqd, throw an error. + var = ""; + if (IsSubVariable(eval, var) != std::string::npos) { + throw std::runtime_error("Variable substitution depth too large: " + + std::to_string(kMaxSubsts) + " " + expr); + } else { + return eval; + } +} + +optional Configuration::GetEnv(const std::string &key) const { + char buf[2048]; + + if ("user.name" == key) { +#ifdef HAVE_GETLOGIN + return std::experimental::make_optional(getlogin()); +#else + DLOG(WARNING) << "Client user.name not implemented"; + return optional(); +#endif + } + + if ("user.dir" == key) { +#ifdef HAVE_GETCWD + if (getcwd(buf, sizeof(buf))) { + return std::experimental::make_optional(buf); + } else { + return optional(); + } +#else + DLOG(WARNING) << "Client user.dir not implemented"; + return optional(); +#endif + } + + if ("user.home" == key) { +#if defined(HAVE_GETUID) && defined(HAVE_GETPWUID_R) + uid = getuid(); + if (!getpwuid_r(uid, &pw, buf, sizeof(buf), &pwp)) { + return std::experimental::make_optional(buf); + } else { + return optional(); + } +#else + DLOG(WARNING) << "Client user.home not implemented"; + return optional(); +#endif + } + return optional(); +} + +optional Configuration::GetProperty(const std::string &key) const { + auto found = hb_property_.find(key); + if (found != hb_property_.end()) { + return std::experimental::make_optional(found->second.value); + } else { + return optional(); + } +} + +optional Configuration::Get(const std::string &key) const { + optional raw = GetProperty(key); + if (raw) { + return std::experimental::make_optional(SubstituteVars(*raw)); + } else { + return optional(); + } +} + +std::string Configuration::Get(const std::string &key, const std::string &default_value) const { + return Get(key).value_or(default_value); +} + +optional Configuration::GetInt(const std::string &key) const { + optional raw = Get(key); + if (raw) { + try { + return std::experimental::make_optional(boost::lexical_cast(*raw)); + } catch (const boost::bad_lexical_cast &blex) { + throw std::runtime_error(blex.what()); + } + } + return optional(); +} + +int32_t Configuration::GetInt(const std::string &key, int32_t default_value) const { + return GetInt(key).value_or(default_value); +} + +optional Configuration::GetLong(const std::string &key) const { + optional raw = Get(key); + if (raw) { + try { + return std::experimental::make_optional(boost::lexical_cast(*raw)); + } catch (const boost::bad_lexical_cast &blex) { + throw std::runtime_error(blex.what()); + } + } + return optional(); +} + +int64_t Configuration::GetLong(const std::string &key, int64_t default_value) const { + return GetLong(key).value_or(default_value); +} + +optional Configuration::GetDouble(const std::string &key) const { + optional raw = Get(key); + if (raw) { + try { + return std::experimental::make_optional(boost::lexical_cast(*raw)); + } catch (const boost::bad_lexical_cast &blex) { + throw std::runtime_error(blex.what()); + } + } + return optional(); +} + +double Configuration::GetDouble(const std::string &key, double default_value) const { + return GetDouble(key).value_or(default_value); +} + +optional Configuration::GetBool(const std::string &key) const { + optional raw = Get(key); + if (raw) { + if (!strcasecmp((*raw).c_str(), "true")) { + return std::experimental::make_optional(true); + } else if (!strcasecmp((*raw).c_str(), "false")) { + return std::experimental::make_optional(false); + } else { + throw std::runtime_error("Unexpected value found while conversion to bool."); + } + } + return optional(); +} + +bool Configuration::GetBool(const std::string &key, bool default_value) const { + return GetBool(key).value_or(default_value); +} + +void Configuration::Set(const std::string &key, const std::string &value) { + hb_property_[key] = value; +} + +void Configuration::SetInt(const std::string &key, int32_t value) { + Set(key, boost::lexical_cast(value)); +} + +void Configuration::SetLong(const std::string &key, int64_t value) { + Set(key, boost::lexical_cast(value)); +} + +void Configuration::SetDouble(const std::string &key, double value) { + Set(key, boost::lexical_cast(value)); +} + +void Configuration::SetBool(const std::string &key, bool value) { + Set(key, boost::lexical_cast(value)); +} + +} /* namespace hbase */ diff --git a/hbase-native-client/common/configuration.h b/hbase-native-client/common/configuration.h new file mode 100644 index 0000000..d70941c --- /dev/null +++ b/hbase-native-client/common/configuration.h @@ -0,0 +1,232 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +/* Some pieces of code have been added from HDFS-8707 */ +#pragma once + +#include +#include +#include + +#include + +namespace hbase { + +template +using optional = std::experimental::optional; + +class Configuration { + public: + /** + * @brief Create Configuration object using an empty map. This + * object WILL NOT be initialized from reading XML configuration (hbase-site.xml, etc). + * Use HBaseConfigurationLoader to initialize the configuration from + * hbase-default.xml and hbase-site.xml files. + */ + Configuration(); + + ~Configuration(); + + /** + * @brief Returns value identified by key in ConfigMap else default value if + * property is absent. + * @param key Property whose value is to be fetched. SubstituteVars will be + * called for any variable expansion. + */ + std::string Get(const std::string &key, const std::string &default_value) const; + + /** + * @brief Returns int32_t identified by key in ConfigMap else default value if + * property is absent. + * @param key Property whose value is to be fetched. Internally Get(key) will + * be called. + * @throws std::runtime_error if conversion to int32_t fails + */ + int32_t GetInt(const std::string &key, int32_t default_value) const; + + /** + * @brief Returns int64_t identified by key in ConfigMap else default value if + * property is absent. + * @param key Property whose value is to be fetched. Internally Get(key) will + * be called and + * @throws std::runtime_error if conversion to int64_t fails + */ + int64_t GetLong(const std::string &key, int64_t default_value) const; + + /** + * @brief Returns double identified by key in ConfigMap else default value if + * property is absent. + * @param key Property whose value is to be fetched. Internally Get(key) will + * be called. + * @throws std::runtime_error if conversion to double fails + */ + double GetDouble(const std::string &key, double default_value) const; + + /** + * @brief Returns bool identified by key in ConfigMap else default value if + * property is absent. + * @param key Property whose value is to be fetched. Internally Get(key) will + * be called. + * @throws std::runtime_error if conversion to bool fails + */ + bool GetBool(const std::string &key, bool default_value) const; + + /** + * @brief This method sets the given key to the value. + * @param key property name + * @param value property value + */ + void Set(const std::string &key, const std::string &value); + + /** + * @brief This method sets the given key to the value. + * @param key property name + * @param value property value + */ + void SetInt(const std::string &key, int32_t value); + + /** + * @brief This method sets the given key to the value. + * @param key property name + * @param value property value + */ + void SetLong(const std::string &key, int64_t value); + + /** + * @brief This method sets the given key to the value. + * @param key property name + * @param value property value + */ + void SetDouble(const std::string &key, double value); + + /** + * @brief This method sets the given key to the value. + * @param key property name + * @param value property value + */ + void SetBool(const std::string &key, bool value); + + private: + friend class HBaseConfigurationLoader; + + /* Transparent data holder for property values */ + /* Same as Jira HDFS-8707 */ + struct ConfigData { + std::string value; + bool final; + ConfigData() : final(false) {} + explicit ConfigData(const std::string &value) : value(value), final(false) {} + void operator=(const std::string &new_value) { + value = new_value; + final = false; + } + }; + + /** + * @brief Map which hold configuration properties. + */ + using ConfigMap = std::map; + + /** + * @brief Create Configuration object using config_map; + * @param config_map - Map consisting of properties. + */ + explicit Configuration(ConfigMap &config_map); + + // Property map filled all the properties loaded by ConfigurationLoader + ConfigMap hb_property_; + + // Variable expansion depth + const int kMaxSubsts = 20; + + /** + * @brief returns value for a property identified by key in ConfigMap. + * SubstituteVars will be called for any variable expansion. + * @param key Key whose value is to be fetched. + */ + optional Get(const std::string &key) const; + + /** + * @brief returns optional int32_t value identified by the key in ConfigMap. + * Get(key) is called to get the string value which is then converted to + * int32_t using boost::lexical_cast. + * @param key Key whose value is to be fetched. + * @throws std::runtime_error if conversion to int32_t fails + */ + optional GetInt(const std::string &key) const; + + /** + * @brief returns optional int64_t value identified by the key in ConfigMap. + * Get(key) is called internally to get the string value which is then + * converted to int64_t using boost::lexical_cast. + * @param key Key whose value is to be fetched. + * @throws std::runtime_error if conversion to int64_t fails + */ + optional GetLong(const std::string &key) const; + + /** + * @brief returns optional double value identified by the key in ConfigMap. + * Get(key) is called to get the string value which is then converted to + * double using boost::lexical_cast. + * @param key Key whose value is to be fetched. + * @throws std::runtime_error if conversion to double fails + */ + optional GetDouble(const std::string &key) const; + + /** + * @brief returns optional bool for a property identified by key in ConfigMap. + * Get(key) is called to get the string value which is then converted to bool + * by checking the validity. + * @param key Key whose value is to be fetched. Get(key) is called to get the + * string value which is then converted to bool. + * @throws std::runtime_error if conversion to bool fails + */ + optional GetBool(const std::string &key) const; + + /** + * @brief This method will perform any variable expansion if present. + * @param expression expression string to perform substitutions on. + * @throws std::runtime_error if MAX_SUBSTS is exhausted and any more + * substitutions are reqd to be performed. + */ + std::string SubstituteVars(const std::string &expr) const; + + /** + * @brief This method will check if variable expansion has to be performed or + * not. Expression will be checked for presence of "${" and "}" to perform + * variable expansion. + * @param expr Expression on which will be checked for validity. + * @param sub_variable Extracted variable from expr which will be checked + * against environment value or ConfigMap values. + */ + size_t IsSubVariable(const std::string &expr, std::string &sub_variable) const; + + /** + * @brief This method will fetch value for key from environment if present. + * @param key key to be fetched from environment. + */ + optional GetEnv(const std::string &key) const; + + /** + * @brief This method will fetch value for key from ConfigMap if present. + * @param key key to be fetched from environment. + */ + optional GetProperty(const std::string &key) const; +}; +} /* namespace hbase */ diff --git a/hbase-native-client/common/hbase_configuration_loader.cc b/hbase-native-client/common/hbase_configuration_loader.cc new file mode 100644 index 0000000..541e04a --- /dev/null +++ b/hbase-native-client/common/hbase_configuration_loader.cc @@ -0,0 +1,208 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "common/hbase_configuration_loader.h" + +#include +#include +#include +#include + +namespace hbase { + +bool is_valid_bool(const std::string &raw) { + if (raw.empty()) { + return false; + } + + if (!strcasecmp(raw.c_str(), "true")) { + return true; + } + if (!strcasecmp(raw.c_str(), "false")) { + return true; + } + return false; +} + +bool str_to_bool(const std::string &raw) { + if (!strcasecmp(raw.c_str(), "true")) { + return true; + } + return false; +} + +HBaseConfigurationLoader::HBaseConfigurationLoader() {} + +HBaseConfigurationLoader::~HBaseConfigurationLoader() {} + +void HBaseConfigurationLoader::SetDefaultSearchPath() { + /* + * Try (in order, taking the first valid one): + * $HBASE_CONF_DIR + * /etc/hbase/conf + * + */ + const char *hadoop_conf_dir_env = getenv("HBASE_CONF"); + if (hadoop_conf_dir_env) { + AddToSearchPath(hadoop_conf_dir_env); + } else { + AddToSearchPath(kHBaseDefauktConfPath); + } +} + +void HBaseConfigurationLoader::ClearSearchPath() { search_paths_.clear(); } + +void HBaseConfigurationLoader::SetSearchPath(const std::string &search_path) { + search_paths_.clear(); + + std::vector paths; + std::string::size_type start = 0; + std::string::size_type end = search_path.find(kSearchPathSeparator); + + while (end != std::string::npos) { + paths.push_back(search_path.substr(start, end - start)); + start = ++end; + end = search_path.find(kSearchPathSeparator, start); + } + paths.push_back(search_path.substr(start, search_path.length())); + + for (auto path : paths) { + AddToSearchPath(path); + } +} + +void HBaseConfigurationLoader::AddToSearchPath(const std::string &search_path) { + if (search_path.empty()) return; + + std::string path_to_add(search_path); + if (search_path.back() != kFileSeparator) { + path_to_add += kFileSeparator; + } + if (std::find(search_paths_.begin(), search_paths_.end(), path_to_add) == search_paths_.end()) + search_paths_.push_back(path_to_add); +} + +void HBaseConfigurationLoader::AddDefaultResources() { + resources_.push_back(kHBaseDefaultXml); + resources_.push_back(kHBaseSiteXml); +} + +void HBaseConfigurationLoader::AddResources(const std::string &filename) { + if (std::find(resources_.begin(), resources_.end(), filename) == resources_.end()) + resources_.push_back(filename); +} + +optional HBaseConfigurationLoader::LoadDefaultResources() { + SetDefaultSearchPath(); + AddDefaultResources(); + ConfigMap conf_property; + bool success = false; + for (auto dir : search_paths_) { + for (auto file : resources_) { + std::string config_file = dir + file; + std::ifstream stream(config_file); + if (stream.is_open()) { + success |= LoadProperties(config_file, conf_property); + } else { + DLOG(WARNING) << "Unable to open file[" << config_file << "]"; + } + } + } + if (success) { + return std::experimental::make_optional(Configuration(conf_property)); + } else { + return optional(); + } +} + +optional HBaseConfigurationLoader::LoadResources( + const std::string &search_path, const std::vector &resources) { + SetSearchPath(search_path); + for (const auto &resource : resources) AddResources(resource); + ConfigMap conf_property; + bool success = false; + for (auto dir : search_paths_) { + for (auto file : resources_) { + std::string config_file = dir + file; + std::ifstream stream(config_file); + if (stream.is_open()) { + success |= LoadProperties(config_file, conf_property); + } else { + DLOG(WARNING) << "Unable to open file[" << config_file << "]"; + } + } + } + if (success) { + return std::experimental::make_optional(Configuration(conf_property)); + } else { + return optional(); + } +} + +bool HBaseConfigurationLoader::LoadProperties(const std::string &file, ConfigMap &property_map) { + // Create empty property tree object + using boost::property_tree::ptree; + ptree pt; + try { + // Load XML file and put contents in a property tree. + // If read fails, throw exception. + read_xml(file, pt); + + // If configuration key is not found exception is thrown + std::string configuration = pt.get("configuration"); + + // Iterate over configuration section. + // Store all found properties in ConfigMap + BOOST_FOREACH (ptree::value_type &v, pt.get_child("configuration")) { + if ("property" == v.first) { + std::string name_node = v.second.get("name"); + std::string value_node = v.second.get("value"); + if ((name_node.size() > 0) && (value_node.size() > 0)) { + boost::optional final_node = v.second.get_optional("final"); + UpdateMapWithValue(property_map, name_node, value_node, final_node); + } + } + } + } catch (std::exception &ex) { + DLOG(WARNING) << "Exception in parsing file [" << file << "]:[" << ex.what() << "]"; + return false; + } + return true; +} + +bool HBaseConfigurationLoader::UpdateMapWithValue(ConfigMap &map, const std::string &key, + const std::string &value, + boost::optional final_text) { + auto map_value = map.find(key); + if (map_value != map.end() && map_value->second.final) { + return false; + } + + bool final_value = false; + if (nullptr != final_text.get_ptr()) { + if (is_valid_bool(final_text.get())) { + final_value = str_to_bool(final_text.get()); + } + } + + map[key].value = value; + map[key].final = final_value; + return true; +} +} /* namespace hbase */ diff --git a/hbase-native-client/common/hbase_configuration_loader.h b/hbase-native-client/common/hbase_configuration_loader.h new file mode 100644 index 0000000..7435798 --- /dev/null +++ b/hbase-native-client/common/hbase_configuration_loader.h @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#pragma once + +#include +#include +#include + +#include +#include + +#include "common/configuration.h" + +namespace hbase { + +template +using optional = std::experimental::optional; + +class HBaseConfigurationLoader { + public: + HBaseConfigurationLoader(); + ~HBaseConfigurationLoader(); + + /** + * @brief Creates a Configuration object based on default resources loaded + * from default search paths. Default search path will be either $HBASE_CONF + * is set or /etc/hbase/conf. Default resources are hbase-default.xml and + * hbase-site.xml.SetDefaultSearchPath() and AddDefaultResources() are used + * for the same. + * Values are loaded in from hbase-default.xml first and then from + * hbase-site.xml. + * Properties in hbase-site.xml will override the ones in hbase-default.xml + * unless marked as final + */ + optional LoadDefaultResources(); + + /* + * @brief Creates a Configuration object based on resources loaded from search + * paths. Search paths are defined in search_path. Values are loaded from + * resources and will be overridden unless marked as final + * @param search_path - ':' search paths to load resources. + * @param resources - list of resources used to load configuration properties. + */ + optional LoadResources(const std::string &search_path, + const std::vector &resources); + + private: + using ConfigMap = Configuration::ConfigMap; + const std::string kHBaseDefaultXml = "hbase-default.xml"; + const std::string kHBaseSiteXml = "hbase-site.xml"; + const std::string kHBaseDefauktConfPath = "/etc/hbase/conf"; + + // Adds FILE_SEPARATOR to the search path + const char kFileSeparator = '/'; + + // Separator using which multiple search paths can be defined. + const char kSearchPathSeparator = ':'; + + /** + * List of paths which will be looked up for loading properties. + */ + std::vector search_paths_; + + /** + * List of files which will be looked up in search_paths_ to load properties. + */ + std::vector resources_; + + /** + * @brief This method sets the search path to the default search path (i.e. + * "$HBASE_CONF" or "/etc/hbase/conf" if HBASE_CONF is absent) + */ + void SetDefaultSearchPath(); + + /** + * @brief Clears out the set search path(s) + */ + void ClearSearchPath(); + + /** + * @brief Sets the search path to ":"-delimited paths, clearing already + * defined values + * @param search_path Single path or ":"-delimited separated paths + */ + void SetSearchPath(const std::string &search_path); + + /** + * @brief Adds an element to the search path if not already present. + * @param search_path Path that will be added to load config values + */ + void AddToSearchPath(const std::string &search_path); + + /** + * @brief This method will add default resources i.e. hbase-default.xml and + * hbase-site.xml to the default search path. + */ + void AddDefaultResources(); + + /** + * @brief Adds resources to list for loading config values. + * @param filename to be added to resources. + */ + void AddResources(const std::string &filename); + + /** + * @brief Loads properties in file identified by file in a map identified by + * property_map + * @param file XML file which defines HBase configuration properties. + * @param property_map Property map representing HBase configuration as key + * value pairs. + * @throws Boost ptree exception if some parsing issue occurs. + */ + bool LoadProperties(const std::string &file, ConfigMap &property_map); + + /** + * @brief This method will create a map of the name and properties. + * @param map Property map to hold configuration properties. + * @param key value of name node. + * @param value value of value node. + * @param final_text value of final node true or false if present + */ + bool UpdateMapWithValue(ConfigMap &map, const std::string &key, const std::string &value, + boost::optional final_text); +}; + +} /* namespace hbase */ diff --git a/hbase-native-client/common/configuration-test.cc b/hbase-native-client/common/configuration-test.cc new file mode 100644 index 0000000..00023f2 --- /dev/null +++ b/hbase-native-client/common/configuration-test.cc @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "common/configuration.h" +#include + +using hbase::Configuration; + +TEST(Configuration, SetGet) { + Configuration conf; + + EXPECT_EQ(conf.Get("foo", "default"), "default"); + conf.Set("foo", "bar"); + EXPECT_EQ(conf.Get("foo", "default"), "bar"); +} + +TEST(Configuration, SetGetInt) { + Configuration conf; + + EXPECT_EQ(conf.GetInt("foo", 0), 0); + conf.SetInt("foo", 42); + EXPECT_EQ(conf.GetInt("foo", 0), 42); +} + +TEST(Configuration, SetGetLong) { + Configuration conf; + + EXPECT_EQ(conf.GetLong("foo", 0), 0); + conf.SetLong("foo", 42); + EXPECT_EQ(conf.GetLong("foo", 0), 42); +} + +TEST(Configuration, SetGetDouble) { + Configuration conf; + + EXPECT_EQ(conf.GetDouble("foo", 0), 0); + conf.SetDouble("foo", 42.0); + EXPECT_EQ(conf.GetDouble("foo", 0), 42.0); +} + +TEST(Configuration, SetGetBool) { + Configuration conf; + + EXPECT_EQ(conf.GetBool("foo", false), false); + conf.SetInt("foo", true); + EXPECT_EQ(conf.GetInt("foo", false), true); +} diff --git a/hbase-native-client/common/hbase_configuration-test.cc b/hbase-native-client/common/hbase_configuration-test.cc new file mode 100644 index 0000000..260c721 --- /dev/null +++ b/hbase-native-client/common/hbase_configuration-test.cc @@ -0,0 +1,374 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include +#include + +#include +#include +#include +#include "common/configuration.h" +#include "common/hbase_configuration_loader.h" + +using namespace hbase; + +const std::string kDefHBaseConfPath("./build/test-data/hbase-configuration-test/conf/"); +const std::string kHBaseConfPath("./build/test-data/hbase-configuration-test/custom-conf/"); + +const std::string kHBaseDefaultXml("hbase-default.xml"); +const std::string kHBaseSiteXml("hbase-site.xml"); + +const std::string kHBaseDefaultXmlData( + "\n\n\n\n\n\nhbase.rootdir\n/root/hbase-docker/apps/hbase/data\ntrue\n\n\n\nhbase.zookeeper.property.datadir\nThis value will be " + "overwritten\nfalse\n\n\n\ndefault-prop\ndefault-value\n\n\n"); +const std::string kHBaseSiteXmlData( + "\n\n\n\n\n\nhbase.rootdir\nThis value will not be be " + "overwritten\n\n\n\nhbase.zookeeper.property.datadir\n/root/hbase-docker/zookeeper\n\n\n\nhbase-client.user.name\n${user.name}\n\n\n\nhbase-client.user.dir\n${user.dir}\n\n\n\nhbase-client.user.home\n${user.home}\n\n\n\nselfRef\n${selfRef}\n\n\n\nfoo.substs\n${bar},${bar},${bar},${bar},${bar},${bar},${bar},${bar},${" + "bar},${bar},\n\n\n\nfoo.substs.exception\n${bar},${bar},${bar},${bar},${bar},${bar},${bar},${bar},${" + "bar},${bar},${bar},${bar},${bar},${bar},${bar},${bar},${bar},${bar},${bar}" + ",${bar},${bar}\n\n\n\nbar\nbar-value\n\n\n\ncustom-prop\ncustom-value\n\n\n\nint\n16000\n\n\n\nint.largevalue\n2147483646\n\n\n\nint.exception\n2147483648\n\n\n\nlong\n2147483850\n\n\n\nlong.largevalue\n9223372036854775807\n\n\n\nlong.exception\n9223372036854775810\n\n\n\ndouble\n17.9769e+100\n\n\n\ndouble.largevalue\n170.769e+200\n\n\n\ndouble.exception\n1.79769e+310\n\n\n\nbool.true\ntrue\n\n\n\nbool.false\nfalse\n\n\n\nbool.exception\nunknown " + "bool\n\n\n"); + +void WriteDataToFile(const std::string &file, const std::string &xml_data) { + std::ofstream hbase_conf; + hbase_conf.open(file.c_str()); + hbase_conf << xml_data; + hbase_conf.close(); +} + +void CreateHBaseConf(const std::string &dir, const std::string &file, const std::string xml_data) { + // Directory will be created if not present + if (!boost::filesystem::exists(dir)) { + boost::filesystem::create_directories(dir); + } + // Remove temp file always + boost::filesystem::remove((dir + file).c_str()); + WriteDataToFile((dir + file), xml_data); +} + +void CreateHBaseConfWithEnv() { + CreateHBaseConf(kDefHBaseConfPath, kHBaseDefaultXml, kHBaseDefaultXmlData); + CreateHBaseConf(kDefHBaseConfPath, kHBaseSiteXml, kHBaseSiteXmlData); + setenv("HBASE_CONF", kDefHBaseConfPath.c_str(), 1); +} + +/* + * Config will be loaded from $HBASE_CONF. We set it @ kDefHBaseConfPath + * Config values will be loaded from hbase-default.xml and hbase-site.xml + * present in the above path. + */ +TEST(Configuration, LoadConfFromDefaultLocation) { + // Remove already configured env if present. + unsetenv("HBASE_CONF"); + CreateHBaseConf(kDefHBaseConfPath, kHBaseDefaultXml, kHBaseDefaultXmlData); + CreateHBaseConf(kDefHBaseConfPath, kHBaseSiteXml, kHBaseSiteXmlData); + setenv("HBASE_CONF", kDefHBaseConfPath.c_str(), 0); + + HBaseConfigurationLoader loader; + hbase::optional conf = loader.LoadDefaultResources(); + ASSERT_TRUE(conf) << "No configuration object present."; + EXPECT_STREQ((*conf).Get("custom-prop", "Set this value").c_str(), "custom-value"); + EXPECT_STREQ((*conf).Get("default-prop", "Set this value").c_str(), "default-value"); +} + +/* + * Config will be loaded from hbase-site.xml defined at + * kHBaseConfPath + */ +TEST(Configuration, LoadConfFromCustomLocation) { + // Remove already configured env if present. + unsetenv("HBASE_CONF"); + CreateHBaseConf(kHBaseConfPath, kHBaseSiteXml, kHBaseSiteXmlData); + + HBaseConfigurationLoader loader; + std::vector resources{kHBaseSiteXml}; + hbase::optional conf = loader.LoadResources(kHBaseConfPath, resources); + ASSERT_TRUE(conf) << "No configuration object present."; + EXPECT_STREQ((*conf).Get("custom-prop", "").c_str(), "custom-value"); + EXPECT_STRNE((*conf).Get("custom-prop", "").c_str(), "some-value"); +} + +/* + * Config will be loaded from hbase-defualt.xml and hbase-site.xml @ + * kDefHBaseConfPath and kHBaseConfPath respectively. + */ +TEST(Configuration, LoadConfFromMultipleLocatons) { + // Remove already configured env if present. + unsetenv("HBASE_CONF"); + CreateHBaseConf(kDefHBaseConfPath, kHBaseDefaultXml, kHBaseDefaultXmlData); + CreateHBaseConf(kDefHBaseConfPath, kHBaseSiteXml, kHBaseSiteXmlData); + CreateHBaseConf(kHBaseConfPath, kHBaseDefaultXml, kHBaseDefaultXmlData); + CreateHBaseConf(kHBaseConfPath, kHBaseSiteXml, kHBaseSiteXmlData); + + HBaseConfigurationLoader loader; + std::string conf_paths = kDefHBaseConfPath + ":" + kHBaseConfPath; + std::vector resources{kHBaseDefaultXml, kHBaseSiteXml}; + hbase::optional conf = loader.LoadResources(conf_paths, resources); + ASSERT_TRUE(conf) << "No configuration object present."; + EXPECT_STREQ((*conf).Get("default-prop", "From hbase-default.xml").c_str(), "default-value"); + EXPECT_STREQ((*conf).Get("custom-prop", "").c_str(), "custom-value"); + EXPECT_STRNE((*conf).Get("custom-prop", "").c_str(), "some-value"); +} + +/* + * Config will be loaded from hbase-defualt.xml and hbase-site.xml @ + * $HBASE_CONF. + * We set HBASE_CONF to kDefHBaseConfPath + * Below tests load the conf files in the same way unless specified. + */ +TEST(Configuration, DefaultValues) { + // Remove already configured env if present. + unsetenv("HBASE_CONF"); + CreateHBaseConfWithEnv(); + + HBaseConfigurationLoader loader; + hbase::optional conf = loader.LoadDefaultResources(); + ASSERT_TRUE(conf) << "No configuration object present."; + EXPECT_STREQ((*conf).Get("default-prop", "Set this value.").c_str(), "default-value"); + EXPECT_STREQ((*conf).Get("custom-prop", "Set this value.").c_str(), "custom-value"); +} + +TEST(Configuration, FinalValues) { + // Remove already configured env if present. + unsetenv("HBASE_CONF"); + CreateHBaseConfWithEnv(); + + HBaseConfigurationLoader loader; + hbase::optional conf = loader.LoadDefaultResources(); + ASSERT_TRUE(conf) << "No configuration object present."; + EXPECT_STREQ((*conf).Get("hbase.rootdir", "").c_str(), "/root/hbase-docker/apps/hbase/data"); + EXPECT_STREQ((*conf).Get("hbase.zookeeper.property.datadir", "").c_str(), + "/root/hbase-docker/zookeeper"); + EXPECT_STRNE((*conf).Get("hbase.rootdir", "").c_str(), "This value will not be be overwritten"); + EXPECT_STRNE((*conf).Get("hbase.zookeeper.property.datadir", "").c_str(), + "This value will be overwritten"); +} + +/* + * Config will be loaded from HBASE_CONF which we set in + * CreateHBaseConfWithEnv(). + * Config values will be loaded from hbase-default.xml and hbase-site.xml in the + * above path. + */ +TEST(Configuration, EnvVars) { + // Remove already configured env if present. + unsetenv("HBASE_CONF"); + CreateHBaseConfWithEnv(); + + HBaseConfigurationLoader loader; + hbase::optional conf = loader.LoadDefaultResources(); + ASSERT_TRUE(conf) << "No configuration object present."; + EXPECT_STREQ((*conf).Get("hbase-client.user.name", "").c_str(), "${user.name}"); + EXPECT_STRNE((*conf).Get("hbase-client.user.name", "root").c_str(), "test-user"); +} + +TEST(Configuration, SelfRef) { + // Remove already configured env if present. + unsetenv("HBASE_CONF"); + CreateHBaseConfWithEnv(); + + HBaseConfigurationLoader loader; + hbase::optional conf = loader.LoadDefaultResources(); + ASSERT_TRUE(conf) << "No configuration object present."; + EXPECT_STREQ((*conf).Get("selfRef", "${selfRef}").c_str(), "${selfRef}"); +} + +TEST(Configuration, VarExpansion) { + // Remove already configured env if present. + unsetenv("HBASE_CONF"); + CreateHBaseConfWithEnv(); + + HBaseConfigurationLoader loader; + hbase::optional conf = loader.LoadDefaultResources(); + ASSERT_TRUE(conf) << "No configuration object present."; + EXPECT_STREQ((*conf).Get("foo.substs", "foo-value").c_str(), + "bar-value,bar-value,bar-value,bar-value,bar-value,bar-value," + "bar-value,bar-value,bar-value,bar-value,"); + EXPECT_STRNE((*conf).Get("foo.substs", "foo-value").c_str(), "bar-value"); +} + +TEST(Configuration, VarExpansionException) { + // Remove already configured env if present. + unsetenv("HBASE_CONF"); + CreateHBaseConfWithEnv(); + + HBaseConfigurationLoader loader; + hbase::optional conf = loader.LoadDefaultResources(); + ASSERT_TRUE(conf) << "No configuration object present."; + ASSERT_THROW((*conf).Get("foo.substs.exception", "foo-value").c_str(), std::runtime_error); +} + +TEST(Configuration, GetInt) { + // Remove already configured env if present. + unsetenv("HBASE_CONF"); + CreateHBaseConfWithEnv(); + + HBaseConfigurationLoader loader; + hbase::optional conf = loader.LoadDefaultResources(); + ASSERT_TRUE(conf) << "No configuration object present."; + EXPECT_EQ(16000, (*conf).GetInt("int", 0)); + EXPECT_EQ(2147483646, (*conf).GetInt("int.largevalue", 0)); +} + +TEST(Configuration, GetLong) { + // Remove already configured env if present. + unsetenv("HBASE_CONF"); + CreateHBaseConfWithEnv(); + + HBaseConfigurationLoader loader; + hbase::optional conf = loader.LoadDefaultResources(); + ASSERT_TRUE(conf) << "No configuration object present."; + EXPECT_EQ(2147483850, (*conf).GetLong("long", 0)); + EXPECT_EQ(9223372036854775807, (*conf).GetLong("long.largevalue", 0)); +} + +TEST(Configuration, GetDouble) { + // Remove already configured env if present. + unsetenv("HBASE_CONF"); + CreateHBaseConfWithEnv(); + + HBaseConfigurationLoader loader; + hbase::optional conf = loader.LoadDefaultResources(); + ASSERT_TRUE(conf) << "No configuration object present."; + EXPECT_DOUBLE_EQ(17.9769e+100, (*conf).GetDouble("double", 0.0)); + EXPECT_DOUBLE_EQ(170.769e+200, (*conf).GetDouble("double.largevalue", 0.0)); +} + +TEST(Configuration, GetBool) { + // Remove already configured env if present. + unsetenv("HBASE_CONF"); + CreateHBaseConfWithEnv(); + + HBaseConfigurationLoader loader; + hbase::optional conf = loader.LoadDefaultResources(); + ASSERT_TRUE(conf) << "No configuration object present."; + EXPECT_EQ(true, (*conf).GetBool("bool.true", true)); + EXPECT_EQ(false, (*conf).GetBool("bool.false", false)); +} + +TEST(Configuration, GetIntException) { + // Remove already configured env if present. + unsetenv("HBASE_CONF"); + CreateHBaseConfWithEnv(); + + HBaseConfigurationLoader loader; + hbase::optional conf = loader.LoadDefaultResources(); + ASSERT_TRUE(conf) << "No configuration object present."; + ASSERT_THROW((*conf).GetInt("int.exception", 0), std::runtime_error); +} + +TEST(Configuration, GetLongException) { + // Remove already configured env if present. + unsetenv("HBASE_CONF"); + CreateHBaseConfWithEnv(); + + HBaseConfigurationLoader loader; + hbase::optional conf = loader.LoadDefaultResources(); + ASSERT_TRUE(conf) << "No configuration object present."; + ASSERT_THROW((*conf).GetLong("long.exception", 0), std::runtime_error); +} + +TEST(Configuration, GetDoubleException) { + // Remove already configured env if present. + unsetenv("HBASE_CONF"); + CreateHBaseConfWithEnv(); + + HBaseConfigurationLoader loader; + hbase::optional conf = loader.LoadDefaultResources(); + ASSERT_TRUE(conf) << "No configuration object present."; + ASSERT_THROW((*conf).GetDouble("double.exception", 0), std::runtime_error); +} + +TEST(Configuration, GetBoolException) { + // Remove already configured env if present. + unsetenv("HBASE_CONF"); + CreateHBaseConfWithEnv(); + + HBaseConfigurationLoader loader; + hbase::optional conf = loader.LoadDefaultResources(); + ASSERT_TRUE(conf) << "No configuration object present."; + ASSERT_THROW((*conf).GetBool("bool.exception", false), std::runtime_error); +}