From ea00e2d39eef2ddd3f8bd26d698e5da844388c91 Mon Sep 17 00:00:00 2001 From: Sudeep Sunthankar Date: Fri, 25 Nov 2016 23:33:45 +1100 Subject: [PATCH] Below changes 1) Chaged filename of configuration_loader.* to hbase_configuration_loader.* 2) Changed the paths to search for hbase configuration files in unit tests to relative ones. 3) Changed line wrapping to 100 4) Changed conditions in unit tests to ASSERT_TRUE() checks. diff --git a/hbase-native-client/core/BUCK b/hbase-native-client/core/BUCK index 7e9044a..9dbc82e 100644 --- a/hbase-native-client/core/BUCK +++ b/hbase-native-client/core/BUCK @@ -29,6 +29,8 @@ cxx_library( "meta-utils.h", "get.h", "time_range.h", + "configuration.h", + "hbase_configuration_loader.h", ], srcs=[ "cell.cc", @@ -37,6 +39,8 @@ cxx_library( "meta-utils.cc", "get.cc", "time_range.cc", + "configuration.cc", + "hbase_configuration_loader.cc", ], deps=[ "//connection:connection", @@ -75,6 +79,12 @@ cxx_test(name="time_range-test", ], deps=[":core", ], run_test_separately=True, ) +cxx_test(name="hbase_configuration-test", + srcs=[ + "hbase_configuration-test.cc", + ], + deps=[":core", ], + run_test_separately=True, ) cxx_binary(name="simple-client", srcs=["simple-client.cc", ], deps=[":core", "//connection:connection"], ) diff --git a/hbase-native-client/core/configuration.cc b/hbase-native-client/core/configuration.cc new file mode 100644 index 0000000..f497872 --- /dev/null +++ b/hbase-native-client/core/configuration.cc @@ -0,0 +1,221 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "configuration.h" + +#include + +#include +#include + +namespace hbase { + +Configuration::Configuration(ConfigMap &config_map) + : hb_property_(std::move(config_map)) { +} + +Configuration::~Configuration() { +} + +size_t Configuration::IsSubVariable(const std::string &expr, std::string &sub_variable) const { + size_t start_pos = expr.find("${"); + if (std::string::npos != start_pos) { + size_t pos_next = expr.find("}", start_pos + 1); + if (std::string::npos != pos_next) { + sub_variable = expr.substr(start_pos + 2, pos_next - (start_pos + 2)); + } + } + return start_pos; +} + +std::string Configuration::SubstituteVars(const std::string &expr) const { + if (0 == expr.size()) + return expr; + + std::string eval(expr); + std::string value_to_be_replaced(""); + std::string var(""); + for (int i = 0; i < kMaxSubsts; i++) { + var = ""; + size_t start_pos = IsSubVariable(eval, var); + if (start_pos != std::string::npos) { + // We are blindly checking for environment property at first. + // If we don't get any value from GetEnv, check in hbase-site.xml. + value_to_be_replaced = GetEnv(var).value_or(GetProperty(var).value_or("")); + + // we haven't found any value yet so we are returning eval + if (0 == value_to_be_replaced.size()) { + return eval; + } + + // return original expression if there is a loop + if (value_to_be_replaced == expr) { + return expr; + } + + eval.replace(start_pos, var.size() + 3, value_to_be_replaced); + + } else { + // No further expansion required. + return eval; + } + } + // We reached here if the loop is exhausted + // If MAX_SUBSTS is exhausted, check if more variable substitution is reqd. + // If any-more substitutions are reqd, throw an error. + var = ""; + if (IsSubVariable(eval, var) != std::string::npos) { + throw std::runtime_error( + "Variable substitution depth too large: " + std::to_string(kMaxSubsts) + " " + expr); + } else { + return eval; + } +} + +optional Configuration::GetEnv(const std::string &key) const { + char buf[2048]; + + if ("user.name" == key) { +#ifdef HAVE_GETLOGIN + return std::experimental::make_optional(getlogin()); +#else + DLOG(WARNING)<< "Client user.name not implemented"; + return optional(); +#endif + } + + if ("user.dir" == key) { +#ifdef HAVE_GETCWD + if (getcwd(buf, sizeof(buf))) { + return std::experimental::make_optional(buf); + } else { + return optional(); + } +#else + DLOG(WARNING)<< "Client user.dir not implemented"; + return optional(); +#endif + } + + if ("user.home" == key) { +#if defined(HAVE_GETUID) && defined(HAVE_GETPWUID_R) + uid = getuid(); + if (!getpwuid_r(uid, &pw, buf, sizeof(buf), &pwp)) { + return std::experimental::make_optional(buf); + } else { + return optional(); + } +#else + DLOG(WARNING)<< "Client user.home not implemented"; + return optional(); +#endif + } + return optional(); +} + +optional Configuration::GetProperty(const std::string &key) const { + + auto found = hb_property_.find(key); + if (found != hb_property_.end()) { + return std::experimental::make_optional(found->second.value); + } else { + return optional(); + } +} + +optional Configuration::Get(const std::string &key) const { + optional raw = GetProperty(key); + if (raw) { + return std::experimental::make_optional(SubstituteVars(*raw)); + } else { + return optional(); + } +} + +std::string Configuration::Get(const std::string &key, const std::string &default_value) const { + return Get(key).value_or(default_value); +} + +optional Configuration::GetInt(const std::string &key) const { + optional raw = Get(key); + if (raw) { + try { + return std::experimental::make_optional(boost::lexical_cast(*raw)); + } catch (const boost::bad_lexical_cast &blex) { + throw std::runtime_error(blex.what()); + } + } + return optional(); +} + +int32_t Configuration::GetInt(const std::string &key, int32_t default_value) const { + return GetInt(key).value_or(default_value); +} + +optional Configuration::GetLong(const std::string &key) const { + optional raw = Get(key); + if (raw) { + try { + return std::experimental::make_optional(boost::lexical_cast(*raw)); + } catch (const boost::bad_lexical_cast &blex) { + throw std::runtime_error(blex.what()); + } + } + return optional(); +} + +int64_t Configuration::GetLong(const std::string &key, int64_t default_value) const { + return GetLong(key).value_or(default_value); +} + +optional Configuration::GetDouble(const std::string &key) const { + optional raw = Get(key); + if (raw) { + try { + return std::experimental::make_optional(boost::lexical_cast(*raw)); + } catch (const boost::bad_lexical_cast &blex) { + throw std::runtime_error(blex.what()); + } + } + return optional(); +} + +double Configuration::GetDouble(const std::string &key, double default_value) const { + return GetDouble(key).value_or(default_value); +} + +optional Configuration::GetBool(const std::string& key) const { + optional raw = Get(key); + if (raw) { + if (!strcasecmp((*raw).c_str(), "true")) { + return std::experimental::make_optional(true); + } else if (!strcasecmp((*raw).c_str(), "false")) { + return std::experimental::make_optional(false); + } else { + throw std::runtime_error("Unexpected value found while conversion to bool."); + } + } + return optional(); +} + +bool Configuration::GetBool(const std::string& key, bool default_value) const { + return GetBool(key).value_or(default_value); +} + +} /* namespace hbase */ diff --git a/hbase-native-client/core/configuration.h b/hbase-native-client/core/configuration.h new file mode 100644 index 0000000..eedcba2 --- /dev/null +++ b/hbase-native-client/core/configuration.h @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +/* Some pieces of code have been added from HDFS-8707 */ +#pragma once + +#include +#include +#include + +#include + +namespace hbase { + +template using optional = std::experimental::optional; + +class Configuration { + public: + ~Configuration(); + + /** + * @brief Returns value identified by key in ConfigMap else default value if property is absent. + * @param key Property whose value is to be fetched. SubstituteVars will be called for any variable expansion. + */ + std::string Get(const std::string &key, const std::string &default_value) const; + + /** + * @brief Returns int32_t identified by key in ConfigMap else default value if property is absent. + * @param key Property whose value is to be fetched. Internally Get(key) will be called. + * @throws std::runtime_error if conversion to int32_t fails + */ + int32_t GetInt(const std::string &key, int32_t default_value) const; + + /** + * @brief Returns int64_t identified by key in ConfigMap else default value if property is absent. + * @param key Property whose value is to be fetched. Internally Get(key) will be called and + * @throws std::runtime_error if conversion to int64_t fails + */ + int64_t GetLong(const std::string &key, int64_t default_value) const; + + /** + * @brief Returns double identified by key in ConfigMap else default value if property is absent. + * @param key Property whose value is to be fetched. Internally Get(key) will be called. + * @throws std::runtime_error if conversion to double fails + */ + double GetDouble(const std::string &key, double default_value) const; + + /** + * @brief Returns bool identified by key in ConfigMap else default value if property is absent. + * @param key Property whose value is to be fetched. Internally Get(key) will be called. + * @throws std::runtime_error if conversion to bool fails + */ + bool GetBool(const std::string &key, bool default_value) const; + + private: + friend class HBaseConfigurationLoader; + + /* Transparent data holder for property values */ + /* Same as Jira HDFS-8707 */ + struct ConfigData { + std::string value; + bool final; + ConfigData() + : final(false) { + } + ; + ConfigData(const std::string &value) + : value(value), + final(false) { + } + void operator=(const std::string &new_value) { + value = new_value; + final = false; + } + }; + + /** + * @brief Map which hold configuration properties. + */ + using ConfigMap = std::map; + + /** + * @brief Create Configuration object using config_map; + * @param config_map - Map consisting of properties. + */ + Configuration(ConfigMap &config_map); + + // Property map filled all the properties loaded by ConfigurationLoader + ConfigMap hb_property_; + + // Variable expansion depth + const int kMaxSubsts = 20; + + /** + * @brief returns value for a property identified by key in ConfigMap. SubstituteVars will be called for any variable expansion. + * @param key Key whose value is to be fetched. + */ + optional Get(const std::string &key) const; + + /** + * @brief returns optional int32_t value identified by the key in ConfigMap. Get(key) is called to get the string value which is then converted to int32_t using boost::lexical_cast. + * @param key Key whose value is to be fetched. + * @throws std::runtime_error if conversion to int32_t fails + */ + optional GetInt(const std::string &key) const; + + /** + * @brief returns optional int64_t value identified by the key in ConfigMap. Get(key) is called internally to get the string value which is then converted to int64_t using boost::lexical_cast. + * @param key Key whose value is to be fetched. + * @throws std::runtime_error if conversion to int64_t fails + */ + optional GetLong(const std::string &key) const; + + /** + * @brief returns optional double value identified by the key in ConfigMap. Get(key) is called to get the string value which is then converted to double using boost::lexical_cast. + * @param key Key whose value is to be fetched. + * @throws std::runtime_error if conversion to double fails + */ + optional GetDouble(const std::string &key) const; + + /** + * @brief returns optional bool for a property identified by key in ConfigMap. Get(key) is called to get the string value which is then converted to bool by checking the validity. + * @param key Key whose value is to be fetched. Get(key) is called to get the string value which is then converted to bool. + * @throws std::runtime_error if conversion to bool fails + */ + optional GetBool(const std::string &key) const; + + /** + * @brief This method will perform any variable expansion if present. + * @param expression expression string to perform substitutions on. + * @throws std::runtime_error if MAX_SUBSTS is exhausted and any more substitutions are reqd to be performed. + */ + std::string SubstituteVars(const std::string &expr) const; + + /** + * @brief This method will check if variable expansion has to be performed or not. Expression will be checked for presence of "${" and "}" to perform variable expansion. + * @param expr Expression on which will be checked for validity. + * @param sub_variable Extracted variable from expr which will be checked against environment value or ConfigMap values. + */ + size_t IsSubVariable(const std::string &expr, std::string &sub_variable) const; + + /** + * @brief This method will fetch value for key from environment if present. + * @param key key to be fetched from environment. + */ + optional GetEnv(const std::string &key) const; + + /** + * @brief This method will fetch value for key from ConfigMap if present. + * @param key key to be fetched from environment. + */ + optional GetProperty(const std::string &key) const; +}; +} /* namespace hbase */ diff --git a/hbase-native-client/core/hbase_configuration-test.cc b/hbase-native-client/core/hbase_configuration-test.cc new file mode 100644 index 0000000..e9f2e15 --- /dev/null +++ b/hbase-native-client/core/hbase_configuration-test.cc @@ -0,0 +1,297 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include +#include + +#include +#include +#include +#include "core/hbase_configuration_loader.h" +#include "core/configuration.h" + +using namespace hbase; + +const std::string kDefHBaseConfPath("./build/conf/"); +const std::string kHBaseConfPath("./build/custom-conf/"); + +const std::string kHBaseDefaultXml("hbase-default.xml"); +const std::string kHBaseSiteXml("hbase-site.xml"); + +const std::string kHBaseDefaultXmlData( + "\n\n\n\n\n\nhbase.rootdir\n/root/hbase-docker/apps/hbase/data\ntrue\n\n\n\nhbase.zookeeper.property.datadir\nThis value will be overwritten\nfalse\n\n\n\ndefault-prop\ndefault-value\n\n\n"); +const std::string kHBaseSiteXmlData( + "\n\n\n\n\n\nhbase.rootdir\nThis value will not be be overwritten\n\n\n\nhbase.zookeeper.property.datadir\n/root/hbase-docker/zookeeper\n\n\n\nhbase-client.user.name\n${user.name}\n\n\n\nhbase-client.user.dir\n${user.dir}\n\n\n\nhbase-client.user.home\n${user.home}\n\n\n\nselfRef\n${selfRef}\n\n\n\nfoo.substs\n${bar},${bar},${bar},${bar},${bar},${bar},${bar},${bar},${bar},${bar},\n\n\n\nfoo.substs.exception\n${bar},${bar},${bar},${bar},${bar},${bar},${bar},${bar},${bar},${bar},${bar},${bar},${bar},${bar},${bar},${bar},${bar},${bar},${bar},${bar},${bar}\n\n\n\nbar\nbar-value\n\n\n\ncustom-prop\ncustom-value\n\n\n\nint\n16000\n\n\n\nint.largevalue\n2147483646\n\n\n\nint.exception\n2147483648\n\n\n\nlong\n2147483850\n\n\n\nlong.largevalue\n9223372036854775807\n\n\n\nlong.exception\n9223372036854775810\n\n\n\ndouble\n17.9769e+100\n\n\n\ndouble.largevalue\n170.769e+200\n\n\n\ndouble.exception\n1.79769e+310\n\n\n\nbool.true\ntrue\n\n\n\nbool.false\nfalse\n\n\n\nbool.exception\nunknown bool\n\n\n"); + +void WriteDataToFile(const std::string &file, const std::string &xml_data) { + std::ofstream hbase_conf; + hbase_conf.open(file.c_str()); + hbase_conf << xml_data; + hbase_conf.close(); +} + +void CreateHBaseConf(const std::string &dir, const std::string &file, const std::string xml_data) { + // Directory will be created if not present + if (!boost::filesystem::exists(dir)) { + boost::filesystem::create_directories(dir); + } + // Remove temp file always + boost::filesystem::remove((dir + file).c_str()); + WriteDataToFile((dir + file), xml_data); +} + +void CreateHBaseConfWithEnv() { + + CreateHBaseConf(kDefHBaseConfPath, kHBaseDefaultXml, kHBaseDefaultXmlData); + CreateHBaseConf(kDefHBaseConfPath, kHBaseSiteXml, kHBaseSiteXmlData); + setenv("HBASE_CONF", kDefHBaseConfPath.c_str(), 1); +} + +/* + * Config will be loaded from $HBASE_CONF. We set it @ /usr/src/hbase/conf/ + * Config values will be loaded from hbase-default.xml and hbase-site.xml present in the above path. + */ +TEST(Configuration, LoadConfFromDefaultLocation) { + //Remove already configured env if present. + unsetenv("HBASE_CONF"); + CreateHBaseConf(kDefHBaseConfPath, kHBaseDefaultXml, kHBaseDefaultXmlData); + CreateHBaseConf(kDefHBaseConfPath, kHBaseSiteXml, kHBaseSiteXmlData); + setenv("HBASE_CONF", kDefHBaseConfPath.c_str(), 0); + + HBaseConfigurationLoader loader; + hbase::optional conf = loader.LoadDefaultResources(); + ASSERT_TRUE(conf)<< "No configuration object present."; + EXPECT_STREQ((*conf).Get("custom-prop", "Set this value").c_str(), "custom-value"); + EXPECT_STREQ((*conf).Get("default-prop", "Set this value").c_str(), "default-value"); +} + +/* + * Config will be loaded from hbase-site.xml defined at /usr/src/hbase/hbase-native-client/build/ + */ +TEST(Configuration, LoadConfFromCustomLocation) { + //Remove already configured env if present. + unsetenv("HBASE_CONF"); + CreateHBaseConf(kHBaseConfPath, kHBaseSiteXml, kHBaseSiteXmlData); + + HBaseConfigurationLoader loader; + std::vector resources { kHBaseSiteXml }; + hbase::optional conf = loader.LoadResources(kHBaseConfPath, resources); + ASSERT_TRUE(conf)<< "No configuration object present."; + EXPECT_STREQ((*conf).Get("custom-prop", "").c_str(), "custom-value"); + EXPECT_STRNE((*conf).Get("custom-prop", "").c_str(), "some-value"); +} + +/* + * Config will be loaded from hbase-defualt.xml and hbase-site.xml @ /usr/src/hbase/conf/ and /usr/src/hbase/hbase-native-client/build/ resp. + */ +TEST(Configuration, LoadConfFromMultipleLocatons) { + //Remove already configured env if present. + unsetenv("HBASE_CONF"); + CreateHBaseConf(kDefHBaseConfPath, kHBaseDefaultXml, kHBaseDefaultXmlData); + CreateHBaseConf(kDefHBaseConfPath, kHBaseSiteXml, kHBaseSiteXmlData); + CreateHBaseConf(kHBaseConfPath, kHBaseDefaultXml, kHBaseDefaultXmlData); + CreateHBaseConf(kHBaseConfPath, kHBaseSiteXml, kHBaseSiteXmlData); + + HBaseConfigurationLoader loader; + std::string conf_paths = kDefHBaseConfPath + ":" + kHBaseConfPath; + std::vector resources { kHBaseDefaultXml, kHBaseSiteXml }; + hbase::optional conf = loader.LoadResources(conf_paths, resources); + ASSERT_TRUE(conf)<< "No configuration object present."; + EXPECT_STREQ((*conf).Get("default-prop", "From hbase-default.xml").c_str(), "default-value"); + EXPECT_STREQ((*conf).Get("custom-prop", "").c_str(), "custom-value"); + EXPECT_STRNE((*conf).Get("custom-prop", "").c_str(), "some-value"); +} + +/* + * Config will be loaded from hbase-defualt.xml and hbase-site.xml @ $HBASE_CONF. + * We set HBASE_CONF to /usr/src/hbase/hbase-native-client/build/ + * Below tests load the conf files in the same way unless specified. + */ +TEST(Configuration, DefaultValues) { + //Remove already configured env if present. + unsetenv("HBASE_CONF"); + CreateHBaseConfWithEnv(); + + HBaseConfigurationLoader loader; + hbase::optional conf = loader.LoadDefaultResources(); + ASSERT_TRUE(conf)<< "No configuration object present."; + EXPECT_STREQ((*conf).Get("default-prop", "Set this value.").c_str(), "default-value"); + EXPECT_STREQ((*conf).Get("custom-prop", "Set this value.").c_str(), "custom-value"); +} + +TEST(Configuration, FinalValues) { + //Remove already configured env if present. + unsetenv("HBASE_CONF"); + CreateHBaseConfWithEnv(); + + HBaseConfigurationLoader loader; + hbase::optional conf = loader.LoadDefaultResources(); + ASSERT_TRUE(conf)<< "No configuration object present."; + EXPECT_STREQ((*conf).Get("hbase.rootdir", "").c_str(), "/root/hbase-docker/apps/hbase/data"); + EXPECT_STREQ((*conf).Get("hbase.zookeeper.property.datadir", "").c_str(), + "/root/hbase-docker/zookeeper"); + EXPECT_STRNE((*conf).Get("hbase.rootdir", "").c_str(), "This value will not be be overwritten"); + EXPECT_STRNE((*conf).Get("hbase.zookeeper.property.datadir", "").c_str(), + "This value will be overwritten"); +} + +/* + * Config will be loaded from HBASE_CONF which we set in CreateHBaseConfWithEnv(). + * Config values will be loaded from hbase-default.xml and hbase-site.xml in the above path. + * + */ +TEST(Configuration, EnvVars) { + //Remove already configured env if present. + unsetenv("HBASE_CONF"); + CreateHBaseConfWithEnv(); + + HBaseConfigurationLoader loader; + hbase::optional conf = loader.LoadDefaultResources(); + ASSERT_TRUE(conf)<< "No configuration object present."; + EXPECT_STREQ((*conf).Get("hbase-client.user.name", "").c_str(), "${user.name}"); + EXPECT_STRNE((*conf).Get("hbase-client.user.name", "root").c_str(), "test-user"); +} + +TEST(Configuration, SelfRef) { + //Remove already configured env if present. + unsetenv("HBASE_CONF"); + CreateHBaseConfWithEnv(); + + HBaseConfigurationLoader loader; + hbase::optional conf = loader.LoadDefaultResources(); + ASSERT_TRUE(conf)<< "No configuration object present."; + EXPECT_STREQ((*conf).Get("selfRef", "${selfRef}").c_str(), "${selfRef}"); +} + +TEST(Configuration, VarExpansion) { + //Remove already configured env if present. + unsetenv("HBASE_CONF"); + CreateHBaseConfWithEnv(); + + HBaseConfigurationLoader loader; + hbase::optional conf = loader.LoadDefaultResources(); + ASSERT_TRUE(conf)<< "No configuration object present."; + EXPECT_STREQ( + (*conf).Get("foo.substs", "foo-value").c_str(), + "bar-value,bar-value,bar-value,bar-value,bar-value,bar-value,bar-value,bar-value,bar-value,bar-value,"); + EXPECT_STRNE((*conf).Get("foo.substs", "foo-value").c_str(), "bar-value"); +} + +TEST(Configuration, VarExpansionException) { + //Remove already configured env if present. + unsetenv("HBASE_CONF"); + CreateHBaseConfWithEnv(); + + HBaseConfigurationLoader loader; + hbase::optional conf = loader.LoadDefaultResources(); + ASSERT_TRUE(conf)<< "No configuration object present."; + ASSERT_THROW((*conf).Get("foo.substs.exception", "foo-value").c_str(), std::runtime_error); +} + +TEST(Configuration, GetInt) { + //Remove already configured env if present. + unsetenv("HBASE_CONF"); + CreateHBaseConfWithEnv(); + + HBaseConfigurationLoader loader; + hbase::optional conf = loader.LoadDefaultResources(); + ASSERT_TRUE(conf)<< "No configuration object present."; + EXPECT_EQ(16000, (*conf).GetInt("int", 0)); + EXPECT_EQ(2147483646, (*conf).GetInt("int.largevalue", 0)); +} + +TEST(Configuration, GetLong) { + //Remove already configured env if present. + unsetenv("HBASE_CONF"); + CreateHBaseConfWithEnv(); + + HBaseConfigurationLoader loader; + hbase::optional conf = loader.LoadDefaultResources(); + ASSERT_TRUE(conf)<< "No configuration object present."; + EXPECT_EQ(2147483850, (*conf).GetLong("long", 0)); + EXPECT_EQ(9223372036854775807, (*conf).GetLong("long.largevalue", 0)); +} + +TEST(Configuration, GetDouble) { + //Remove already configured env if present. + unsetenv("HBASE_CONF"); + CreateHBaseConfWithEnv(); + + HBaseConfigurationLoader loader; + hbase::optional conf = loader.LoadDefaultResources(); + ASSERT_TRUE(conf)<< "No configuration object present."; + EXPECT_DOUBLE_EQ(17.9769e+100, (*conf).GetDouble("double", 0.0)); + EXPECT_DOUBLE_EQ(170.769e+200, (*conf).GetDouble("double.largevalue", 0.0)); +} + +TEST(Configuration, GetBool) { + //Remove already configured env if present. + unsetenv("HBASE_CONF"); + CreateHBaseConfWithEnv(); + + HBaseConfigurationLoader loader; + hbase::optional conf = loader.LoadDefaultResources(); + ASSERT_TRUE(conf)<< "No configuration object present."; + EXPECT_EQ(true, (*conf).GetBool("bool.true", true)); + EXPECT_EQ(false, (*conf).GetBool("bool.false", false)); +} + +TEST(Configuration, GetIntException) { + //Remove already configured env if present. + unsetenv("HBASE_CONF"); + CreateHBaseConfWithEnv(); + + HBaseConfigurationLoader loader; + hbase::optional conf = loader.LoadDefaultResources(); + ASSERT_TRUE(conf)<< "No configuration object present."; + ASSERT_THROW((*conf).GetInt("int.exception", 0), std::runtime_error); +} + +TEST(Configuration, GetLongException) { + //Remove already configured env if present. + unsetenv("HBASE_CONF"); + CreateHBaseConfWithEnv(); + + HBaseConfigurationLoader loader; + hbase::optional conf = loader.LoadDefaultResources(); + ASSERT_TRUE(conf)<< "No configuration object present."; + ASSERT_THROW((*conf).GetLong("long.exception", 0), std::runtime_error); +} + +TEST(Configuration, GetDoubleException) { + //Remove already configured env if present. + unsetenv("HBASE_CONF"); + CreateHBaseConfWithEnv(); + + HBaseConfigurationLoader loader; + hbase::optional conf = loader.LoadDefaultResources(); + ASSERT_TRUE(conf)<< "No configuration object present."; + ASSERT_THROW((*conf).GetDouble("double.exception", 0), std::runtime_error); +} + +TEST(Configuration, GetBoolException) { + //Remove already configured env if present. + unsetenv("HBASE_CONF"); + CreateHBaseConfWithEnv(); + + HBaseConfigurationLoader loader; + hbase::optional conf = loader.LoadDefaultResources(); + ASSERT_TRUE(conf)<< "No configuration object present."; + ASSERT_THROW((*conf).GetBool("bool.exception", false), std::runtime_error); +} diff --git a/hbase-native-client/core/hbase_configuration_loader.cc b/hbase-native-client/core/hbase_configuration_loader.cc new file mode 100644 index 0000000..ec57196 --- /dev/null +++ b/hbase-native-client/core/hbase_configuration_loader.cc @@ -0,0 +1,215 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "core/hbase_configuration_loader.h" + +#include +#include +#include +#include + +namespace hbase { + +bool is_valid_bool(const std::string& raw) { + if (raw.empty()) { + return false; + } + + if (!strcasecmp(raw.c_str(), "true")) { + return true; + } + if (!strcasecmp(raw.c_str(), "false")) { + return true; + } + return false; +} + +bool str_to_bool(const std::string& raw) { + if (!strcasecmp(raw.c_str(), "true")) { + return true; + } + return false; +} + +HBaseConfigurationLoader::HBaseConfigurationLoader() { +} + +HBaseConfigurationLoader::~HBaseConfigurationLoader() { +} + +void HBaseConfigurationLoader::SetDefaultSearchPath() { + /* + * Try (in order, taking the first valid one): + * $HBASE_CONF_DIR + * /etc/hbase/conf + * + */ + const char * hadoop_conf_dir_env = getenv("HBASE_CONF"); + if (hadoop_conf_dir_env) { + AddToSearchPath(hadoop_conf_dir_env); + } else { + AddToSearchPath(kHBaseDefauktConfPath); + } +} + +void HBaseConfigurationLoader::ClearSearchPath() { + search_paths_.clear(); +} + +void HBaseConfigurationLoader::SetSearchPath(const std::string & search_path) { + search_paths_.clear(); + + std::vector paths; + std::string::size_type start = 0; + std::string::size_type end = search_path.find(kSearchPathSeparator); + + while (end != std::string::npos) { + paths.push_back(search_path.substr(start, end - start)); + start = ++end; + end = search_path.find(kSearchPathSeparator, start); + } + paths.push_back(search_path.substr(start, search_path.length())); + + for (auto path : paths) { + AddToSearchPath(path); + } +} + +void HBaseConfigurationLoader::AddToSearchPath(const std::string & search_path) { + if (search_path.empty()) + return; + + std::string path_to_add(search_path); + if (search_path.back() != kFileSeparator) { + path_to_add += kFileSeparator; + } + if (std::find(search_paths_.begin(), search_paths_.end(), path_to_add) == search_paths_.end()) + search_paths_.push_back(path_to_add); +} + +void HBaseConfigurationLoader::AddDefaultResources() { + resources_.push_back(kHBaseDefaultXml); + resources_.push_back(kHBaseSiteXml); +} + +void HBaseConfigurationLoader::AddResources(const std::string &filename) { + if (std::find(resources_.begin(), resources_.end(), filename) == resources_.end()) + resources_.push_back(filename); +} + +optional HBaseConfigurationLoader::LoadDefaultResources() { + SetDefaultSearchPath(); + AddDefaultResources(); + ConfigMap conf_property; + bool success = false; + for (auto dir : search_paths_) { + for (auto file : resources_) { + std::string config_file = dir + file; + std::ifstream stream(config_file); + if (stream.is_open()) { + success |= LoadProperties(config_file, conf_property); + } else { + DLOG(WARNING)<< "Unable to open file[" << config_file << "]"; + } + } + } + if (success) { + return std::experimental::make_optional < Configuration > (conf_property); + } else { + return optional(); + } +} + +optional HBaseConfigurationLoader::LoadResources( + const std::string &search_path, const std::vector &resources) { + SetSearchPath(search_path); + for (const auto &resource : resources) + AddResources(resource); + ConfigMap conf_property; + bool success = false; + for (auto dir : search_paths_) { + for (auto file : resources_) { + std::string config_file = dir + file; + std::ifstream stream(config_file); + if (stream.is_open()) { + success |= LoadProperties(config_file, conf_property); + } else { + DLOG(WARNING)<< "Unable to open file[" << config_file << "]"; + } + } + } + if (success) { + return std::experimental::make_optional < Configuration > (conf_property); + } else { + return optional(); + } +} + +bool HBaseConfigurationLoader::LoadProperties(const std::string &file, ConfigMap &property_map) { + // Create empty property tree object + using boost::property_tree::ptree; + ptree pt; + try { + // Load XML file and put contents in a property tree. + // If read fails, throw exception. + read_xml(file, pt); + + // If configuration key is not found exception is thrown + std::string configuration = pt.get("configuration"); + + // Iterate over configuration section. + // Store all found properties in ConfigMap + BOOST_FOREACH(ptree::value_type & v, pt.get_child("configuration")) { + if ("property" == v.first) { + std::string name_node = v.second.get("name"); + std::string value_node = v.second.get("value"); + if ((name_node.size() > 0) && (value_node.size() > 0)) { + boost::optional final_node = v.second.get_optional("final"); + UpdateMapWithValue(property_map, name_node, value_node, final_node); + } + } + } + } catch (std::exception &ex) { + DLOG(WARNING)<< "Exception in parsing file [" << file << "]:[" << ex.what() + << "]"; + return false; + } + return true; +} + +bool HBaseConfigurationLoader::UpdateMapWithValue(ConfigMap & map, const std::string& key, + const std::string& value, + boost::optional final_text) { + auto map_value = map.find(key); + if (map_value != map.end() && map_value->second.final) { + return false; + } + + bool final_value = false; + if (nullptr != final_text.get_ptr()) { + if (is_valid_bool(final_text.get())) { + final_value = str_to_bool(final_text.get()); + } + } + + map[key].value = value; + map[key].final = final_value; + return true; +} +} /* namespace hbase */ diff --git a/hbase-native-client/core/hbase_configuration_loader.h b/hbase-native-client/core/hbase_configuration_loader.h new file mode 100644 index 0000000..46a9df5 --- /dev/null +++ b/hbase-native-client/core/hbase_configuration_loader.h @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#pragma once + +#include +#include +#include + +#include +#include + +#include "core/configuration.h" + +namespace hbase { + +template using optional = std::experimental::optional; + +class HBaseConfigurationLoader { + + public: + HBaseConfigurationLoader(); + ~HBaseConfigurationLoader(); + + /** + * @brief Creates a Configuration object based on default resources loaded from default search paths. Default search path will be either $HBASE_CONF is set or /etc/hbase/conf. Default resources are hbase-default.xml and hbase-site.xml.SetDefaultSearchPath() and AddDefaultResources() are used for the same. + * Values are loaded in from hbase-default.xml first and then from hbase-site.xml. + * Properties in hbase-site.xml will override the ones in hbase-default.xml unless marked as final + */ + optional LoadDefaultResources(); + + /* + * @brief Creates a Configuration object based on resources loaded from search paths. Search paths are defined in search_path. Values are loaded from resources and will be overridden unless marked as final + * @param search_path - ':' search paths to load resources. + * @param resources - list of resources used to load configuration properties. + */ + optional LoadResources(const std::string &search_path, + const std::vector &resources); + + private: + using ConfigMap = Configuration::ConfigMap; + const std::string kHBaseDefaultXml = "hbase-default.xml"; + const std::string kHBaseSiteXml = "hbase-site.xml"; + const std::string kHBaseDefauktConfPath = "/etc/hbase/conf"; + + // Adds FILE_SEPARATOR to the search path + const char kFileSeparator = '/'; + + // Separator using which multiple search paths can be defined. + const char kSearchPathSeparator = ':'; + + /** + * List of paths which will be looked up for loading properties. + */ + std::vector search_paths_; + + /** + * List of files which will be looked up in search_paths_ to load properties. + */ + std::vector resources_; + + /** + * @brief This method sets the search path to the default search path (i.e. "$HBASE_CONF" or "/etc/hbase/conf" if HBASE_CONF is absent) + */ + void SetDefaultSearchPath(); + + /** + * @brief Clears out the set search path(s) + */ + void ClearSearchPath(); + + /** + * @brief Sets the search path to ":"-delimited paths, clearing already defined values + * @param search_path Single path or ":"-delimited separated paths + */ + void SetSearchPath(const std::string & search_path); + + /** + * @brief Adds an element to the search path if not already present. + * @param search_path Path that will be added to load config values + */ + void AddToSearchPath(const std::string & search_path); + + /** + * @brief This method will add default resources i.e. hbase-default.xml and hbase-site.xml to the default search path. + */ + void AddDefaultResources(); + + /** + * @brief Adds resources to list for loading config values. + * @param filename to be added to resources. + */ + void AddResources(const std::string &filename); + + /** + * @brief Loads properties in file identified by file in a map identified by property_map + * @param file XML file which defines HBase configuration properties. + * @param property_map Property map representing HBase configuration as key value pairs. + * @throws Boost ptree exception if some parsing issue occurs. + */ + bool LoadProperties(const std::string &file, ConfigMap &property_map); + + /** + * @brief This method will create a map of the name and properties. + * @param map Property map to hold configuration properties. + * @param key value of name node. + * @param value value of value node. + * @param final_text value of final node true or false if present + */ + bool UpdateMapWithValue(ConfigMap& map, const std::string& key, const std::string& value, + boost::optional final_text); + +}; + +} /* namespace hbase */ -- 1.8.3.1