From 0264d3643b3b699dd71ac9631f5dd2389cff9457 Mon Sep 17 00:00:00 2001 From: Sudeep Sunthankar Date: Wed, 9 Nov 2016 21:19:22 +1100 Subject: [PATCH] 1) Changes hbase_configuration.cc/.h to configuration.cc/.h 2) Made ConfigData struct, member of Configuration class, which will be populated by ConfigurationLoader 3) boost::optional changed to std::experimental::optional diff --git a/hbase-native-client/core/BUCK b/hbase-native-client/core/BUCK index 7e9044a..5a4896c 100644 --- a/hbase-native-client/core/BUCK +++ b/hbase-native-client/core/BUCK @@ -29,6 +29,8 @@ cxx_library( "meta-utils.h", "get.h", "time_range.h", + "configuration.h", + "configuration_loader.h", ], srcs=[ "cell.cc", @@ -37,6 +39,8 @@ cxx_library( "meta-utils.cc", "get.cc", "time_range.cc", + "configuration.cc", + "configuration_loader.cc", ], deps=[ "//connection:connection", @@ -75,6 +79,12 @@ cxx_test(name="time_range-test", ], deps=[":core", ], run_test_separately=True, ) +cxx_test(name="configuration-test", + srcs=[ + "configuration-test.cc", + ], + deps=[":core", ], + run_test_separately=True, ) cxx_binary(name="simple-client", srcs=["simple-client.cc", ], deps=[":core", "//connection:connection"], ) diff --git a/hbase-native-client/core/configuration-test.cc b/hbase-native-client/core/configuration-test.cc new file mode 100644 index 0000000..2a39f7a --- /dev/null +++ b/hbase-native-client/core/configuration-test.cc @@ -0,0 +1,365 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include +#include + +#include +#include +#include +#include "core/configuration_loader.h" +#include "core/configuration.h" + +using namespace hbase; + +const std::string DEF_HBASE_CONF_PATH("/usr/src/hbase/conf/"); +const std::string HBASE_CONF_PATH("/usr/src/hbase/hbase-native-client/build/"); +const std::string HBASE_DEFAULT_XML("hbase-default.xml"); +const std::string HBASE_SITE_XML("hbase-site.xml"); + +void WriteDataToFile(const std::string &file, const std::string &xml_data) { + std::ofstream hbase_conf; + hbase_conf.open(file.c_str()); + hbase_conf << xml_data; + hbase_conf.close(); +} + +void CreateTestHbaseXml(const std::string &config_file, + bool require_hbase_conf_env = true, + bool require_hbase_default_xml = true) { + // Remove the temp files at first before every test. + if (require_hbase_default_xml) { + boost::filesystem::remove((HBASE_CONF_PATH + HBASE_DEFAULT_XML).c_str()); + } + boost::filesystem::remove((HBASE_CONF_PATH + config_file).c_str()); + + // We are setting HBASE_CONF env variable which'll be used to load conf files. + if (require_hbase_conf_env) { + setenv("HBASE_CONF", HBASE_CONF_PATH.c_str(), 1); + } + + std::string xml_data( + "\n\n\n\n\n\nhbase.rootdir\nThis value will not be be overwritten\n\n\n\nhbase.zookeeper.property.datadir\n/root/hbase-docker/zookeeper\n\n\n\nhbase-client.user.name\n${user.name}\n\n\n\nhbase-client.user.dir\n${user.dir}\n\n\n\nhbase-client.user.home\n${user.home}\n\n\n\nselfRef\n${selfRef}\n\n\n\nfoo.substs\n${bar},${bar},${bar},${bar},${bar},${bar},${bar},${bar},${bar},${bar},\n\n\n\nfoo.substs.exception\n${bar},${bar},${bar},${bar},${bar},${bar},${bar},${bar},${bar},${bar},${bar},${bar},${bar},${bar},${bar},${bar},${bar},${bar},${bar},${bar},${bar}\n\n\n\nbar\nbar-value\n\n\n\ncustom-prop\ncustom-value\n\n\n\nint\n16000\n\n\n\nint.largevalue\n2147483646\n\n\n\nint.exception\n2147483648\n\n\n\nlong\n2147483850\n\n\n\nlong.largevalue\n9223372036854775807\n\n\n\nlong.exception\n9223372036854775810\n\n\n\ndouble\n17.9769e+100\n\n\n\ndouble.largevalue\n170.769e+200\n\n\n\ndouble.exception\n1.79769e+310\n\n\n\nbool.true\ntrue\n\n\n\nbool.false\nfalse\n\n\n\nbool.exception\nunknown bool\n\n\n"); + + // Create hbase-default.xml + if (require_hbase_default_xml) { + std::string xml_data( + "\n\n\n\n\n\nhbase.rootdir\n/root/hbase-docker/apps/hbase/data\ntrue\n\n\n\nhbase.zookeeper.property.datadir\nThis value will be overwritten\nfalse\n\n\n\ndefault-prop\ndefault-value\n\n\n"); + WriteDataToFile((HBASE_CONF_PATH + HBASE_DEFAULT_XML), xml_data); + } + + // Create hbase-site.xml + WriteDataToFile((HBASE_CONF_PATH + config_file), xml_data); + + return; +} + +/* + * Config will be loaded from HBASE_CONF if it is set else /etc/hbase/conf. + * Config values will be loaded from hbase-default.xml and hbase-site.xml present in the above path. + */ +TEST(Configuration, DefaultConf) { + setenv("HBASE_CONF", DEF_HBASE_CONF_PATH.c_str(), 0); + ASSERT_NO_THROW(Configuration()); + ASSERT_NO_THROW(ConfigurationLoader()); + Configuration conf; + ConfigurationLoader loader; + ASSERT_NO_THROW(loader.SetDefaultSearchPath()); + ASSERT_NO_THROW(loader.AddDefaultResources()); + ASSERT_NO_THROW(loader.Load(conf)); + ASSERT_NO_THROW(conf.Get("custom-prop", "")); + EXPECT_STRNE(conf.Get("custom-prop", "").c_str(), "custom-value"); + EXPECT_STREQ(conf.Get("custom-prop", "Set this value").c_str(), + "Set this value"); +} + +/* + * No HBASE_CONF environment variable set at run-time. + * Config will be loaded from HBASE_CONF if it is set else /etc/hbase/conf. + * We will then clear the default search path and add HBASE_CONF_PATH. + * Search Path will be HBASE_CONF_PATH. + * Config values will be loaded from hbase-site.xml in the above path. + */ +TEST(Configuration, CustomConf) { + CreateTestHbaseXml(HBASE_SITE_XML, false, false); + + ASSERT_NO_THROW(Configuration conf = Configuration()); + ASSERT_NO_THROW(ConfigurationLoader()); + Configuration conf; + ConfigurationLoader loader; + ASSERT_NO_THROW(loader.SetDefaultSearchPath()); + ASSERT_NO_THROW(loader.AddDefaultResources()); + ASSERT_NO_THROW(loader.ClearSearchPath()); + ASSERT_NO_THROW(loader.SetSearchPath(HBASE_CONF_PATH)); + ASSERT_NO_THROW(loader.AddResources(HBASE_SITE_XML)); + ASSERT_NO_THROW(loader.Load(conf)); + ASSERT_NO_THROW(conf.Get("custom-prop", "")); + EXPECT_STREQ(conf.Get("custom-prop", "").c_str(), "custom-value"); + EXPECT_STRNE(conf.Get("custom-prop", "").c_str(), "some-value"); +} + +/* + * No HBASE_CONF environment variable set at run-time. + * Config will be loaded from HBASE_CONF if it is set else /etc/hbase/conf. + * We will then add HBASE_CONF_PATH. + * Search Path will be HBASE_CONF:HBASE_CONF_PATH or /etc/hbase/conf:HBASE_CONF_PATH. + * Config values will be loaded from hbase-default.xml and hbase-site.xml in the above path. + * Below tests load the conf files in the same way unless specified. + * + */ +TEST(Configuration, DefaultAndCustomConf) { + CreateTestHbaseXml(HBASE_SITE_XML, false); + + ASSERT_NO_THROW(Configuration conf = Configuration()); + ASSERT_NO_THROW(ConfigurationLoader()); + Configuration conf; + ConfigurationLoader loader; + ASSERT_NO_THROW(loader.SetDefaultSearchPath()); + ASSERT_NO_THROW(loader.AddDefaultResources()); + ASSERT_NO_THROW(loader.AddToSearchPath(HBASE_CONF_PATH)); + ASSERT_NO_THROW(loader.AddResources(HBASE_DEFAULT_XML)); + ASSERT_NO_THROW(loader.AddResources(HBASE_SITE_XML)); + ASSERT_NO_THROW(loader.Load(conf)); + ASSERT_NO_THROW(conf.Get("default-prop", "From hbase-default.xml")); + ASSERT_NO_THROW(conf.Get("custom-prop", "From hbase-default.xml")); + EXPECT_STREQ(conf.Get("default-prop", "From hbase-default.xml").c_str(), + "default-value"); + EXPECT_STREQ(conf.Get("custom-prop", "From hbase-site.xml").c_str(), + "custom-value"); +} + +TEST(Configuration, DefaultValues) { + CreateTestHbaseXml(HBASE_SITE_XML, false); + + ASSERT_NO_THROW(Configuration conf = Configuration()); + ASSERT_NO_THROW(ConfigurationLoader()); + Configuration conf; + ConfigurationLoader loader; + ASSERT_NO_THROW(loader.SetDefaultSearchPath()); + ASSERT_NO_THROW(loader.AddDefaultResources()); + ASSERT_NO_THROW(loader.AddToSearchPath(HBASE_CONF_PATH)); + ASSERT_NO_THROW(loader.AddResources(HBASE_DEFAULT_XML)); + ASSERT_NO_THROW(loader.AddResources(HBASE_SITE_XML)); + ASSERT_NO_THROW(loader.Load(conf)); + ASSERT_NO_THROW( + conf.Get("property-not-set", "Value for property not present")); + EXPECT_STRNE( + conf.Get("property-not-set", "Value for property not present").c_str(), + "default-value"); + EXPECT_STREQ( + conf.Get("property-not-set", "Value for property not present").c_str(), + "Value for property not present"); +} + +TEST(Configuration, FinalValues) { + CreateTestHbaseXml(HBASE_SITE_XML, false); + + ASSERT_NO_THROW(Configuration()); + ASSERT_NO_THROW(ConfigurationLoader()); + Configuration conf; + ConfigurationLoader loader; + ASSERT_NO_THROW(loader.SetDefaultSearchPath()); + ASSERT_NO_THROW(loader.AddDefaultResources()); + ASSERT_NO_THROW(loader.AddToSearchPath(HBASE_CONF_PATH)); + ASSERT_NO_THROW(loader.AddResources(HBASE_DEFAULT_XML)); + ASSERT_NO_THROW(loader.AddResources(HBASE_SITE_XML)); + ASSERT_NO_THROW(loader.Load(conf)); + ASSERT_NO_THROW(conf.Get("hbase.rootdir", "")); + EXPECT_STREQ(conf.Get("hbase.rootdir", "").c_str(), + "/root/hbase-docker/apps/hbase/data"); + EXPECT_STREQ(conf.Get("hbase.zookeeper.property.datadir", "").c_str(), + "/root/hbase-docker/zookeeper"); + EXPECT_STRNE(conf.Get("hbase.rootdir", "").c_str(), + "This value will not be be overwritten"); + EXPECT_STRNE(conf.Get("hbase.zookeeper.property.datadir", "").c_str(), + "This value will be overwritten"); +} + +/* + * Config will be loaded from HBASE_CONF which we set in CreateTestHbaseXml(). + * Config values will be loaded from hbase-default.xml and hbase-site.xml in the above path. + * Below tests load the conf files in the same way unless specified. + */ +TEST(Configuration, EnvVars) { + CreateTestHbaseXml(HBASE_SITE_XML); + + ASSERT_NO_THROW(Configuration()); + ASSERT_NO_THROW(ConfigurationLoader()); + Configuration conf; + ConfigurationLoader loader; + ASSERT_NO_THROW(loader.SetDefaultSearchPath()); + ASSERT_NO_THROW(loader.AddDefaultResources()); + loader.Load(conf); + ASSERT_NO_THROW(conf.Get("hbase-client.user.name", "")); + EXPECT_STREQ(conf.Get("hbase-client.user.name", "").c_str(), "${user.name}"); + EXPECT_STRNE(conf.Get("hbase-client.user.name", "root").c_str(), "test-user"); +} + +TEST(Configuration, SelfRef) { + CreateTestHbaseXml(HBASE_SITE_XML); + + ASSERT_NO_THROW(Configuration()); + ASSERT_NO_THROW(ConfigurationLoader()); + Configuration conf; + ConfigurationLoader loader; + ASSERT_NO_THROW(loader.SetDefaultSearchPath()); + ASSERT_NO_THROW(loader.AddDefaultResources()); + loader.Load(conf); + ASSERT_NO_THROW(conf.Get("selfRef", "${selfRef}")); + EXPECT_STREQ(conf.Get("selfRef", "${selfRef}").c_str(), "${selfRef}"); +} + +TEST(Configuration, VarExpansion) { + CreateTestHbaseXml(HBASE_SITE_XML); + + ASSERT_NO_THROW(Configuration()); + ASSERT_NO_THROW(ConfigurationLoader()); + Configuration conf; + ConfigurationLoader loader; + ASSERT_NO_THROW(loader.SetDefaultSearchPath()); + ASSERT_NO_THROW(loader.AddDefaultResources()); + loader.Load(conf); + ASSERT_NO_THROW(conf.Get("foo.substs", "foo-value").c_str()); + EXPECT_STREQ( + conf.Get("foo.substs", "foo-value").c_str(), + "bar-value,bar-value,bar-value,bar-value,bar-value,bar-value,bar-value,bar-value,bar-value,bar-value,"); + EXPECT_STRNE(conf.Get("foo.substs", "foo-value").c_str(), "bar-value"); +} + +TEST(Configuration, VarExpansionException) { + CreateTestHbaseXml(HBASE_SITE_XML); + + ASSERT_NO_THROW(Configuration()); + ASSERT_NO_THROW(ConfigurationLoader()); + Configuration conf; + ConfigurationLoader loader; + ASSERT_NO_THROW(loader.SetDefaultSearchPath()); + ASSERT_NO_THROW(loader.AddDefaultResources()); + loader.Load(conf); + ASSERT_THROW(conf.Get("foo.substs.exception", "foo-value").c_str(), + std::runtime_error); +} + +TEST(Configuration, GetInt) { + CreateTestHbaseXml(HBASE_SITE_XML); + + ASSERT_NO_THROW(Configuration()); + ASSERT_NO_THROW(ConfigurationLoader()); + Configuration conf; + ConfigurationLoader loader; + ASSERT_NO_THROW(loader.SetDefaultSearchPath()); + ASSERT_NO_THROW(loader.AddDefaultResources()); + loader.Load(conf); + ASSERT_NO_THROW(conf.GetInt("int", 0)); + ASSERT_NO_THROW(conf.GetInt("int.largevalue", 0)); +} + +TEST(Configuration, GetLong) { + CreateTestHbaseXml(HBASE_SITE_XML); + + ASSERT_NO_THROW(Configuration()); + ASSERT_NO_THROW(ConfigurationLoader()); + Configuration conf; + ConfigurationLoader loader; + ASSERT_NO_THROW(loader.SetDefaultSearchPath()); + ASSERT_NO_THROW(loader.AddDefaultResources()); + loader.Load(conf); + ASSERT_NO_THROW(conf.GetLong("long", 0)); + ASSERT_NO_THROW(conf.GetLong("long.largevalue", 0)); +} + +TEST(Configuration, GetDouble) { + CreateTestHbaseXml(HBASE_SITE_XML); + + ASSERT_NO_THROW(Configuration()); + ASSERT_NO_THROW(ConfigurationLoader()); + Configuration conf; + ConfigurationLoader loader; + ASSERT_NO_THROW(loader.SetDefaultSearchPath()); + ASSERT_NO_THROW(loader.AddDefaultResources()); + loader.Load(conf); + ASSERT_NO_THROW(conf.GetDouble("double", 0.0)); + ASSERT_NO_THROW(conf.GetDouble("double.largevalue", 0.0)); +} + +TEST(Configuration, GetBool) { + CreateTestHbaseXml(HBASE_SITE_XML); + + ASSERT_NO_THROW(Configuration()); + ASSERT_NO_THROW(ConfigurationLoader()); + Configuration conf; + ConfigurationLoader loader; + ASSERT_NO_THROW(loader.SetDefaultSearchPath()); + ASSERT_NO_THROW(loader.AddDefaultResources()); + loader.Load(conf); + ASSERT_NO_THROW(conf.GetBool("bool.true", true)); + ASSERT_NO_THROW(conf.GetBool("bool.false", false)); +} + +TEST(Configuration, GetIntException) { + CreateTestHbaseXml(HBASE_SITE_XML); + + ASSERT_NO_THROW(Configuration()); + ASSERT_NO_THROW(ConfigurationLoader()); + Configuration conf; + ConfigurationLoader loader; + ASSERT_NO_THROW(loader.SetDefaultSearchPath()); + ASSERT_NO_THROW(loader.AddDefaultResources()); + loader.Load(conf); + ASSERT_THROW(conf.GetInt("int.exception", 0), std::runtime_error); +} + +TEST(Configuration, GetLongException) { + CreateTestHbaseXml(HBASE_SITE_XML); + + ASSERT_NO_THROW(Configuration()); + ASSERT_NO_THROW(ConfigurationLoader()); + Configuration conf; + ConfigurationLoader loader; + ASSERT_NO_THROW(loader.SetDefaultSearchPath()); + ASSERT_NO_THROW(loader.AddDefaultResources()); + loader.Load(conf); + ASSERT_THROW(conf.GetLong("long.exception", 0), std::runtime_error); +} + +TEST(Configuration, GetDoubleException) { + CreateTestHbaseXml(HBASE_SITE_XML); + + ASSERT_NO_THROW(Configuration()); + ASSERT_NO_THROW(ConfigurationLoader()); + Configuration conf; + ConfigurationLoader loader; + ASSERT_NO_THROW(loader.SetDefaultSearchPath()); + ASSERT_NO_THROW(loader.AddDefaultResources()); + loader.Load(conf); + ASSERT_THROW(conf.GetDouble("double.exception", 0), std::runtime_error); +} + +TEST(Configuration, GetBoolException) { + CreateTestHbaseXml(HBASE_SITE_XML); + + ASSERT_NO_THROW(Configuration()); + ASSERT_NO_THROW(ConfigurationLoader()); + Configuration conf; + ConfigurationLoader loader; + ASSERT_NO_THROW(loader.SetDefaultSearchPath()); + ASSERT_NO_THROW(loader.AddDefaultResources()); + loader.Load(conf); + ASSERT_THROW(conf.GetBool("bool.exception", false), std::runtime_error); +} diff --git a/hbase-native-client/core/configuration.cc b/hbase-native-client/core/configuration.cc new file mode 100644 index 0000000..db7c8dc --- /dev/null +++ b/hbase-native-client/core/configuration.cc @@ -0,0 +1,226 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "configuration.h" + +#include + +#include +#include + +namespace hbase { + +Configuration::Configuration() {} + +Configuration::~Configuration() {} + +size_t Configuration::IsSubVariable(const std::string &expr, + std::string &sub_variable) const { + size_t start_pos = expr.find("${"); + if (std::string::npos != start_pos) { + size_t pos_next = expr.find("}", start_pos + 1); + if (std::string::npos != pos_next) { + sub_variable = expr.substr(start_pos + 2, pos_next - (start_pos + 2)); + } + } + return start_pos; +} + +std::string Configuration::SubstituteVars(const std::string &expr) const { + if (0 == expr.size()) + return expr; + + std::string eval(expr); + std::string value_to_be_replaced(""); + std::string var(""); + for (int i = 0; i < MAX_SUBSTS; i++) { + var = ""; + size_t start_pos = IsSubVariable(eval, var); + if (start_pos != std::string::npos) { + // We are blindly checking for environment property at first. + // If we don't get any value from GetEnv, check in hbase-site.xml. + value_to_be_replaced = GetEnv(var).value_or( + GetProperty(var).value_or("")); + + // we haven't found any value yet so we are returning eval + if (0 == value_to_be_replaced.size()) { + return eval; + } + + // return original expression if there is a loop + if (value_to_be_replaced == expr) { + return expr; + } + + eval.replace(start_pos, var.size() + 3, value_to_be_replaced); + + } else { + // No further expansion required. + return eval; + } + } + // We reached here if the loop is exhausted + // If MAX_SUBSTS is exhausted, check if more variable substitution is reqd. + // If any-more substitutions are reqd, throw an error. + var = ""; + if (IsSubVariable(eval, var) != std::string::npos) { + throw std::runtime_error( + "Variable substitution depth too large: " + std::to_string(MAX_SUBSTS) + + " " + expr); + } else { + return eval; + } +} + +optional Configuration::GetEnv(const std::string &key) const { + char buf[2048]; + + if ("user.name" == key) { +#ifdef HAVE_GETLOGIN + return std::experimental::make_optional(getlogin()); +#else + DLOG(WARNING) << "Client user.name not implemented"; + return optional(); +#endif + } + + if ("user.dir" == key) { +#ifdef HAVE_GETCWD + if (getcwd(buf, sizeof(buf))) { + return std::experimental::make_optional(buf); + } else { + return optional(); + } +#else + DLOG(WARNING) << "Client user.dir not implemented"; + return optional(); +#endif + } + + if ("user.home" == key) { +#if defined(HAVE_GETUID) && defined(HAVE_GETPWUID_R) + uid = getuid(); + if (!getpwuid_r(uid, &pw, buf, sizeof(buf), &pwp)) { + return std::experimental::make_optional(buf); + } else { + return optional(); + } +#else + DLOG(WARNING) << "Client user.home not implemented"; + return optional(); +#endif + } + return optional(); +} + +optional Configuration::GetProperty(const std::string &key) const { + + auto found = hb_property_.find(key); + if (found != hb_property_.end()) { + return std::experimental::make_optional(found->second.value); + } else { + return optional(); + } +} + +optional Configuration::Get(const std::string &key) const { + optional raw = GetProperty(key); + if(raw){ + return std::experimental::make_optional(SubstituteVars(*raw)); + }else{ + return optional(); + } +} + +std::string Configuration::Get(const std::string &key, + const std::string &default_value) const { + return Get(key).value_or(default_value); +} + +optional Configuration::GetInt(const std::string &key) const { + optional raw = Get(key); + if (raw) { + try { + return std::experimental::make_optional(boost::lexical_cast(*raw)); + } catch (const boost::bad_lexical_cast &blex) { + throw std::runtime_error(blex.what()); + } + } + return optional(); +} + +int32_t Configuration::GetInt(const std::string &key, + int32_t default_value) const { + return GetInt(key).value_or(default_value); +} + +optional Configuration::GetLong(const std::string &key) const { + optional raw = Get(key); + if (raw) { + try { + return std::experimental::make_optional(boost::lexical_cast(*raw)); + } catch (const boost::bad_lexical_cast &blex) { + throw std::runtime_error(blex.what()); + } + } + return optional(); +} + +int64_t Configuration::GetLong(const std::string &key, + int64_t default_value) const { + return GetLong(key).value_or(default_value); +} + +optional Configuration::GetDouble(const std::string &key) const { + optional raw = Get(key); + if (raw) { + try { + return std::experimental::make_optional(boost::lexical_cast(*raw)); + } catch (const boost::bad_lexical_cast &blex) { + throw std::runtime_error(blex.what()); + } + } + return optional(); +} + +double Configuration::GetDouble(const std::string &key, + double default_value) const { + return GetDouble(key).value_or(default_value); +} + +optional Configuration::GetBool(const std::string& key) const { + optional raw = Get(key); + if (raw) { + if (!strcasecmp((*raw).c_str(), "true")) { + return std::experimental::make_optional(true); + } else if (!strcasecmp((*raw).c_str(), "false")) { + return std::experimental::make_optional(false); + } else { + throw std::runtime_error( + "Unexpected value found while conversion to bool."); + } + } + return optional(); +} + +bool Configuration::GetBool(const std::string& key, bool default_value) const { + return GetBool(key).value_or(default_value); +} + +} /* namespace hbase */ diff --git a/hbase-native-client/core/configuration.h b/hbase-native-client/core/configuration.h new file mode 100644 index 0000000..36f84b6 --- /dev/null +++ b/hbase-native-client/core/configuration.h @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +/* Some pieces of code have been added from HDFS-8707 */ +#pragma once + +#include +#include +#include + +#include + +namespace hbase { + +template using optional = std::experimental::optional; + +class Configuration { + public: + Configuration(); + ~Configuration(); + + /** + * @brief Returns value identified by key in CONFIG_MAP else default value if property is absent. + * @param key Property whose value is to be fetched. SubstituteVars will be called for any variable expansion. + */ + std::string Get(const std::string &key, + const std::string &default_value) const; + + /** + * @brief Returns int32_t identified by key in CONFIG_MAP else default value if property is absent. + * @param key Property whose value is to be fetched. Internally Get(key) will be called. + * @throws std::runtime_error if conversion to int32_t fails + */ + int32_t GetInt(const std::string &key, int32_t default_value) const; + + /** + * @brief Returns int64_t identified by key in CONFIG_MAP else default value if property is absent. + * @param key Property whose value is to be fetched. Internally Get(key) will be called and + * @throws std::runtime_error if conversion to int64_t fails + */ + int64_t GetLong(const std::string &key, int64_t default_value) const; + + /** + * @brief Returns double identified by key in CONFIG_MAP else default value if property is absent. + * @param key Property whose value is to be fetched. Internally Get(key) will be called. + * @throws std::runtime_error if conversion to double fails + */ + double GetDouble(const std::string &key, double default_value) const; + + /** + * @brief Returns bool identified by key in CONFIG_MAP else default value if property is absent. + * @param key Property whose value is to be fetched. Internally Get(key) will be called. + * @throws std::runtime_error if conversion to bool fails + */ + bool GetBool(const std::string &key, bool default_value) const; + + private: + /* Transparent data holder for property values */ + /* Same as Jira HDFS-8707 */ + struct ConfigData { + std::string value; + bool final; + ConfigData() + : final(false) { + } + ; + ConfigData(const std::string &value) + : value(value), + final(false) { + } + void operator=(const std::string &new_value) { + value = new_value; + final = false; + } + }; + /* + * @brief Map which hold configuration properties. + */ + typedef std::map ConfigMap; + using CONFIG_MAP = Configuration::ConfigMap; + + // This will enable ConfigurationLoader to access and fill hb_property_ map + friend class ConfigurationLoader; + + // Property map filled all the properties loaded by ConfigurationLoader + CONFIG_MAP hb_property_; + + // Variable expansion depth + const int MAX_SUBSTS = 20; + + /** + * @brief returns value for a property identified by key in CONFIG_MAP. SubstituteVars will be called for any variable expansion. + * @param key Key whose value is to be fetched. + */ + optional Get(const std::string &key) const; + + /** + * @brief returns optional int32_t value identified by the key in CONFIG_MAP. Get(key) is called to get the string value which is then converted to int32_t using boost::lexical_cast. + * @param key Key whose value is to be fetched. + * @throws std::runtime_error if conversion to int32_t fails + */ + optional GetInt(const std::string &key) const; + + /** + * @brief returns optional int64_t value identified by the key in CONFIG_MAP. Get(key) is called internally to get the string value which is then converted to int64_t using boost::lexical_cast. + * @param key Key whose value is to be fetched. + * @throws std::runtime_error if conversion to int64_t fails + */ + optional GetLong(const std::string &key) const; + + /** + * @brief returns optional double value identified by the key in CONFIG_MAP. Get(key) is called to get the string value which is then converted to double using boost::lexical_cast. + * @param key Key whose value is to be fetched. + * @throws std::runtime_error if conversion to double fails + */ + optional GetDouble(const std::string &key) const; + + /** + * @brief returns optional bool for a property identified by key in CONFIG_MAP. Get(key) is called to get the string value which is then converted to bool by checking the validity. + * @param key Key whose value is to be fetched. Get(key) is called to get the string value which is then converted to bool. + * @throws std::runtime_error if conversion to bool fails + */ + optional GetBool(const std::string &key) const; + + /** + * @brief This method will perform any variable expansion if present. + * @param expression expression string to perform substitutions on. + * @throws std::runtime_error if MAX_SUBSTS is exhausted and any more substitutions are reqd to be performed. + */ + std::string SubstituteVars(const std::string &expr) const; + + /** + * @brief This method will check if variable expansion has to be performed or not. Expression will be checked for presence of "${" and "}" to perform variable expansion. + * @param expr Expression on which will be checked for validity. + * @param sub_variable Extracted variable from expr which will be checked against environment value or CONFIG_MAP values. + */ + size_t IsSubVariable(const std::string &expr, + std::string &sub_variable) const; + + /** + * @brief This method will fetch value for key from environment if present. + * @param key key to be fetched from environment. + */ + optional GetEnv(const std::string &key) const; + + /** + * @brief This method will fetch value for key from CONFIG_MAP if present. + * @param key key to be fetched from environment. + */ + optional GetProperty(const std::string &key) const; +}; +} /* namespace hbase */ diff --git a/hbase-native-client/core/configuration_loader.cc b/hbase-native-client/core/configuration_loader.cc new file mode 100644 index 0000000..3050752 --- /dev/null +++ b/hbase-native-client/core/configuration_loader.cc @@ -0,0 +1,187 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "core/configuration_loader.h" + +#include +#include +#include +#include + +namespace hbase { + +bool is_valid_bool(const std::string& raw) { + if (raw.empty()) { + return false; + } + + if (!strcasecmp(raw.c_str(), "true")) { + return true; + } + if (!strcasecmp(raw.c_str(), "false")) { + return true; + } + return false; +} + +bool str_to_bool(const std::string& raw) { + if (!strcasecmp(raw.c_str(), "true")) { + return true; + } + return false; +} + +ConfigurationLoader::ConfigurationLoader() { +} + +ConfigurationLoader::~ConfigurationLoader() { +} + +void ConfigurationLoader::SetDefaultSearchPath() { + /* + * Try (in order, taking the first valid one): + * $HBASE_CONF_DIR + * /etc/hbase/conf + * + */ + const char * hadoop_conf_dir_env = getenv("HBASE_CONF"); + if (hadoop_conf_dir_env) { + AddToSearchPath(hadoop_conf_dir_env); + } else { + AddToSearchPath(HBASE_DEFAULT_CONF_PATH); + } +} + +void ConfigurationLoader::ClearSearchPath() { + search_paths_.clear(); +} + +void ConfigurationLoader::SetSearchPath(const std::string & search_path) { + search_paths_.clear(); + + std::vector paths; + std::string::size_type start = 0; + std::string::size_type end = search_path.find(SEARCH_PATH_SEPARATOR); + + while (end != std::string::npos) { + paths.push_back(search_path.substr(start, end - start)); + start = ++end; + end = search_path.find(SEARCH_PATH_SEPARATOR, start); + } + paths.push_back(search_path.substr(start, search_path.length())); + + for (auto path : paths) { + AddToSearchPath(path); + } +} + +void ConfigurationLoader::AddToSearchPath(const std::string & search_path) { + if (search_path.empty()) + return; + + std::string path_to_add(search_path); + if (search_path.back() != FILE_SEPARATOR) { + path_to_add += FILE_SEPARATOR; + } + if (std::find(search_paths_.begin(), search_paths_.end(), path_to_add) + == search_paths_.end()) + search_paths_.push_back(path_to_add); +} + +void ConfigurationLoader::AddDefaultResources() { + resources_.push_back(HBASE_DEFAULT_XML); + resources_.push_back(HBASE_SITE_XML); +} + +void ConfigurationLoader::AddResources(const std::string &filename) { + if (std::find(resources_.begin(), resources_.end(), filename) + == resources_.end()) + resources_.push_back(filename); +} + +bool ConfigurationLoader::Load(Configuration &conf) { + bool success = false; + for (auto dir : search_paths_) { + for (auto file : resources_) { + std::string config_file = dir + file; + std::ifstream stream(config_file); + if (stream.is_open()) { + success |= LoadProperties(config_file, conf.hb_property_); + } else { + DLOG(WARNING) << "Unable to open file[" << config_file << "]"; + } + } + } + return success; +} + +bool ConfigurationLoader::LoadProperties(const std::string &file, + CONFIG_MAP &property_map) { + // Create empty property tree object + using boost::property_tree::ptree; + ptree pt; + try { + // Load XML file and put contents in a property tree. + // If read fails, throw exception. + read_xml(file, pt); + + // If configuration key is not found exception is thrown + std::string configuration = pt.get("configuration"); + + // Iterate over configuration section. + // Store all found properties in CONFIG_MAP + BOOST_FOREACH(ptree::value_type & v, pt.get_child("configuration")) { + if ("property" == v.first) { + std::string name_node = v.second.get("name"); + std::string value_node = v.second.get("value"); + if ((name_node.size() > 0) && (value_node.size() > 0)) { + boost::optional final_node = v.second + .get_optional("final"); + UpdateMapWithValue(property_map, name_node, value_node, final_node); + } + } + } + } catch (std::exception &ex) { + DLOG(WARNING) << "Exception in parsing file [" << file << "]:[" << ex.what() + << "]"; + return false; + } + return true; +} + +bool ConfigurationLoader::UpdateMapWithValue( + CONFIG_MAP & map, const std::string& key, const std::string& value, + boost::optional final_text) { + auto map_value = map.find(key); + if (map_value != map.end() && map_value->second.final) { + return false; + } + + bool final_value = false; + if (nullptr != final_text.get_ptr()) { + if (is_valid_bool(final_text.get())) { + final_value = str_to_bool(final_text.get()); + } + } + + map[key].value = value; + map[key].final = final_value; + return true; +} +} /* namespace hbase */ diff --git a/hbase-native-client/core/configuration_loader.h b/hbase-native-client/core/configuration_loader.h new file mode 100644 index 0000000..16e8985 --- /dev/null +++ b/hbase-native-client/core/configuration_loader.h @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#pragma once + +#include +#include +#include + +#include +#include + +#include "core/configuration.h" + +namespace hbase { + +template using optional = std::experimental::optional; + +/** + * @brief We set the search path to environment variable HBASE_CONF if present, else /etc/hbase/conf. + * Once set we load hbase-default.xml and hbase-site.xml files present. + * Config values are loaded in CONFIG_MAP from hbase-default.xml first. + * If any property is overridden in hbase-site.xml, CONFIG_MAP will be updated unless marked as final + */ +class ConfigurationLoader { + + public: + ConfigurationLoader(); + ~ConfigurationLoader(); + + /** + * @brief This method sets the search path to the default search path (i.e. "$HBASE_CONF" or "/etc/hbase/conf" if HBASE_CONF is absent) + */ + void SetDefaultSearchPath(); + + /** + * @brief Clears out the set search path(s) + */ + void ClearSearchPath(); + + /** + * @brief Sets the search path to ":"-delimited paths, clearing already defined values + * @param search_path Single path or ":"-delimited separated paths + */ + void SetSearchPath(const std::string & search_path); + + /** + * @brief Adds an element to the search path if not already present. + * @param search_path Path that will be added to load config values + */ + void AddToSearchPath(const std::string & search_path); + + /** + * @brief This method will add default resources i.e. hbase-default.xml and hbase-site.xml to the default search path. + */ + void AddDefaultResources(); + + /** + * @brief Adds resources to list for loading config values. + * @param filename to be added to resources. + */ + void AddResources(const std::string &filename); + + /* + * @brief Loads CONFIG_MAP with values from the config resources present on the search path(s) + */ + bool Load(Configuration &conf); + + /** + * @brief This method will fetch value for key from CONFIG_MAP. + * @param key key to be searched in CONFIG_MAP. + */ + + private: + using CONFIG_MAP = Configuration::ConfigMap; + const std::string HBASE_DEFAULT_XML = "hbase-default.xml"; + const std::string HBASE_SITE_XML = "hbase-site.xml"; + const std::string HBASE_DEFAULT_CONF_PATH="/etc/hbase/conf"; + + // Adds FILE_SEPARATOR to the search path + const char FILE_SEPARATOR = '/'; + + // Separator using which multiple search paths can be defined. + const char SEARCH_PATH_SEPARATOR = ':'; + + /** + * List of paths which will be looked up for loading properties. + */ + std::vector search_paths_; + + /** + * List of files which will be looked up in search_paths_ to load properties. + */ + std::vector resources_; + + /** + * @brief Loads properties in file identified by file in a map identified by property_map + * @param file XML file which defines HBase configuration properties. + * @param property_map Property map representing HBase configuration as key value pairs. + * @throws Boost ptree exception if some parsing issue occurs. + */ + bool LoadProperties(const std::string &file, CONFIG_MAP &property_map); + + /** + * @brief This method will perform any variable substitution if present. + * @param map Property map to hold configuration properties. + * @param key value of name node. + * @param value value of value node. + * @param final_text value of final node true or false if present + */ + bool UpdateMapWithValue(CONFIG_MAP& map, const std::string& key, + const std::string& value, + boost::optional final_text); +}; + +} /* namespace hbase */ -- 1.8.3.1