From c3f5cb3700ad60a9f866fdf4b4b60baafb8f54e4 Mon Sep 17 00:00:00 2001 From: Alexander Kolbasov Date: Tue, 17 Jul 2018 18:06:34 -0700 Subject: [PATCH 1/1] HIVE-20196: Separate MetastoreConf into common and server parts --- .../java/org/apache/hadoop/hive/conf/HiveConf.java | 2 +- .../hadoop/hive/metastore/HiveMetaStore.java | 13 ++- .../hadoop/hive/metastore/conf/MetastoreConf.java | 37 ------ .../hive/metastore/conf/MetastoreServerConf.java | 130 +++++++++++++++++++++ .../metastore/security/DelegationTokenTool.java | 4 +- .../hadoop/hive/metastore/MetaStoreTestUtils.java | 7 +- .../hadoop/hive/metastore/TestHiveMetaStore.java | 4 +- 7 files changed, 150 insertions(+), 47 deletions(-) create mode 100644 standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreServerConf.java diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 4ed1636591..e008331fd7 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -908,7 +908,7 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal METASTORE_TOKEN_SIGNATURE("hive.metastore.token.signature", "", "The delegation token service name to match when selecting a token from the current user's tokens."), /** - * @deprecated Use MetastoreConf.DELEGATION_TOKEN_STORE_CLS + * @deprecated Use MetastoreServerConf.DELEGATION_TOKEN_STORE_CLS */ @Deprecated METASTORE_CLUSTER_DELEGATION_TOKEN_STORE_CLS("hive.cluster.delegation.token.store.class", diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 47f819b60a..9dd306833f 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -79,6 +79,7 @@ import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.common.TableName; import org.apache.hadoop.hive.metastore.api.*; +import org.apache.hadoop.hive.metastore.conf.MetastoreServerConf; import org.apache.hadoop.hive.metastore.events.AddForeignKeyEvent; import org.apache.hadoop.hive.metastore.events.AcidWriteEvent; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; @@ -528,7 +529,8 @@ public void init() throws MetaException { singleInitListener.onInit(context); } - String alterHandlerName = MetastoreConf.getVar(conf, ConfVars.ALTER_HANDLER); + String alterHandlerName = MetastoreServerConf.getVar(conf, + MetastoreServerConf.ConfVars.ALTER_HANDLER); alterHandler = ReflectionUtils.newInstance(JavaUtils.getClass( alterHandlerName, AlterHandler.class), conf); wh = new Warehouse(conf); @@ -581,7 +583,8 @@ public void init() throws MetaException { if (alwaysThreadsInitialized.compareAndSet(false, true)) { ThreadPool.initialize(conf); Collection taskNames = - MetastoreConf.getStringCollection(conf, ConfVars.TASK_THREADS_ALWAYS); + MetastoreServerConf.getStringCollection(conf, + MetastoreServerConf.ConfVars.TASK_THREADS_ALWAYS); for (String taskName : taskNames) { MetastoreTaskThread task = JavaUtils.newInstance(JavaUtils.getClass(taskName, MetastoreTaskThread.class)); @@ -5164,7 +5167,8 @@ private void alter_table_core(final String catName, final String dbname, final S private StorageSchemaReader getStorageSchemaReader() throws MetaException { if (storageSchemaReader == null) { String className = - MetastoreConf.getVar(conf, MetastoreConf.ConfVars.STORAGE_SCHEMA_READER_IMPL); + MetastoreServerConf.getVar(conf, + MetastoreServerConf.ConfVars.STORAGE_SCHEMA_READER_IMPL); Class readerClass = JavaUtils.getClass(className, StorageSchemaReader.class); try { @@ -9284,7 +9288,8 @@ private static void startRemoteOnlyTasks(Configuration conf) throws Exception { ThreadPool.initialize(conf); Collection taskNames = - MetastoreConf.getStringCollection(conf, ConfVars.TASK_THREADS_REMOTE_ONLY); + MetastoreServerConf.getStringCollection(conf, + MetastoreServerConf.ConfVars.TASK_THREADS_REMOTE_ONLY); for (String taskName : taskNames) { MetastoreTaskThread task = JavaUtils.newInstance(JavaUtils.getClass(taskName, MetastoreTaskThread.class)); diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java index c2bbba5813..1b5a2aaaeb 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java @@ -19,17 +19,6 @@ import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.metastore.DefaultStorageSchemaReader; -import org.apache.hadoop.hive.metastore.HiveAlterHandler; -import org.apache.hadoop.hive.metastore.MaterializationsRebuildLockCleanerTask; -import org.apache.hadoop.hive.metastore.MetastoreTaskThread; -import org.apache.hadoop.hive.metastore.RuntimeStatsCleanerTask; -import org.apache.hadoop.hive.metastore.events.EventCleanerTask; -import org.apache.hadoop.hive.metastore.security.MetastoreDelegationTokenManager; -import org.apache.hadoop.hive.metastore.txn.AcidCompactionHistoryService; -import org.apache.hadoop.hive.metastore.txn.AcidHouseKeeperService; -import org.apache.hadoop.hive.metastore.txn.AcidOpenTxnsCounterService; -import org.apache.hadoop.hive.metastore.txn.AcidWriteSetService; import org.apache.hadoop.hive.metastore.utils.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -259,9 +248,6 @@ public static ConfVars getMetaConf(String name) { AGGREGATE_STATS_CACHE_TTL("metastore.aggregate.stats.cache.ttl", "hive.metastore.aggregate.stats.cache.ttl", 600, TimeUnit.SECONDS, "Number of seconds for a cached node to be active in the cache before they become stale."), - ALTER_HANDLER("metastore.alter.handler", "hive.metastore.alter.impl", - HiveAlterHandler.class.getName(), - "Alter handler. For now defaults to the Hive one. Really need a better default option"), ASYNC_LOG_ENABLED("metastore.async.log.enabled", "hive.async.log.enabled", true, "Whether to enable Log4j2's asynchronous logging. Asynchronous logging can give\n" + " significant performance improvement as logging will be handled in separate thread\n" + @@ -435,9 +421,6 @@ public static ConfVars getMetaConf(String name) { "hive.cluster.delegation.token.max-lifetime", 7, TimeUnit.DAYS, ""), DELEGATION_TOKEN_RENEW_INTERVAL("metastore.cluster.delegation.token.renew-interval", "hive.cluster.delegation.token.renew-interval", 1, TimeUnit.DAYS, ""), - DELEGATION_TOKEN_STORE_CLS("metastore.cluster.delegation.token.store.class", - "hive.cluster.delegation.token.store.class", MetastoreDelegationTokenManager.class.getName(), - "Class to store delegation tokens"), DETACH_ALL_ON_COMMIT("javax.jdo.option.DetachAllOnCommit", "javax.jdo.option.DetachAllOnCommit", true, "Detaches all objects from session so that they can be used after transaction is committed"), @@ -744,10 +727,6 @@ public static ConfVars getMetaConf(String name) { STATS_AUTO_UPDATE_WORKER_COUNT("metastore.stats.auto.analyze.worker.count", "hive.metastore.stats.auto.analyze.worker.count", 1, "Number of parallel analyze commands to run for background stats update."), - STORAGE_SCHEMA_READER_IMPL("metastore.storage.schema.reader.impl", "metastore.storage.schema.reader.impl", - DefaultStorageSchemaReader.class.getName(), - "The class to use to read schemas from storage. It must implement " + - "org.apache.hadoop.hive.metastore.StorageSchemaReader"), STORE_MANAGER_TYPE("datanucleus.storeManagerType", "datanucleus.storeManagerType", "rdbms", "metadata store type"), STRICT_MANAGED_TABLES("metastore.strict.managed.tables", "hive.strict.managed.tables", false, "Whether strict managed tables mode is enabled. With this mode enabled, " + @@ -758,22 +737,6 @@ public static ConfVars getMetaConf(String name) { + "When it is set to false, only [a-zA-Z_0-9]+ are supported.\n" + "The only supported special character right now is '/'. This flag applies only to quoted table names.\n" + "The default value is true."), - TASK_THREADS_ALWAYS("metastore.task.threads.always", "metastore.task.threads.always", - EventCleanerTask.class.getName() + "," + RuntimeStatsCleanerTask.class.getName() + "," + - "org.apache.hadoop.hive.metastore.repl.DumpDirCleanerTask" + "," + - "org.apache.hadoop.hive.metastore.HiveProtoEventsCleanerTask", - "Comma separated list of tasks that will be started in separate threads. These will " + - "always be started, regardless of whether the metastore is running in embedded mode " + - "or in server mode. They must implement " + MetastoreTaskThread.class.getName()), - TASK_THREADS_REMOTE_ONLY("metastore.task.threads.remote", "metastore.task.threads.remote", - AcidHouseKeeperService.class.getName() + "," + - AcidOpenTxnsCounterService.class.getName() + "," + - AcidCompactionHistoryService.class.getName() + "," + - AcidWriteSetService.class.getName() + "," + - MaterializationsRebuildLockCleanerTask.class.getName(), - "Command separated list of tasks that will be started in separate threads. These will be" + - " started only when the metastore is running as a separate service. They must " + - "implement " + MetastoreTaskThread.class.getName()), TCP_KEEP_ALIVE("metastore.server.tcp.keepalive", "hive.metastore.server.tcp.keepalive", true, "Whether to enable TCP keepalive for the metastore server. Keepalive will prevent accumulation of half-open connections."), diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreServerConf.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreServerConf.java new file mode 100644 index 0000000000..f5bfbd135b --- /dev/null +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreServerConf.java @@ -0,0 +1,130 @@ +package org.apache.hadoop.hive.metastore.conf; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.DefaultStorageSchemaReader; +import org.apache.hadoop.hive.metastore.HiveAlterHandler; +import org.apache.hadoop.hive.metastore.MaterializationsRebuildLockCleanerTask; +import org.apache.hadoop.hive.metastore.MetastoreTaskThread; +import org.apache.hadoop.hive.metastore.RuntimeStatsCleanerTask; +import org.apache.hadoop.hive.metastore.events.EventCleanerTask; +import org.apache.hadoop.hive.metastore.security.MetastoreDelegationTokenManager; +import org.apache.hadoop.hive.metastore.txn.AcidCompactionHistoryService; +import org.apache.hadoop.hive.metastore.txn.AcidHouseKeeperService; +import org.apache.hadoop.hive.metastore.txn.AcidOpenTxnsCounterService; +import org.apache.hadoop.hive.metastore.txn.AcidWriteSetService; +import org.apache.hadoop.hive.metastore.utils.StringUtils; + +import java.util.Collection; +import java.util.Collections; + +/** + * Version of MetastoreConf class that holds variables that are only used by standalone] + * metastore server. It is needed to avoid classpath dependency on server-specific classes. + * See {@link MetastoreConf} for the description of the structure of this class. + */ +public class MetastoreServerConf { + public enum ConfVars { + // alpha order, PLEASE! + ALTER_HANDLER("metastore.alter.handler", "hive.metastore.alter.impl", + HiveAlterHandler.class.getName(), + "Alter handler. For now defaults to the Hive one. Really need a better default option"), + DELEGATION_TOKEN_STORE_CLS("metastore.cluster.delegation.token.store.class", + "hive.cluster.delegation.token.store.class", MetastoreDelegationTokenManager.class.getName(), + "Class to store delegation tokens"), + TASK_THREADS_ALWAYS("metastore.task.threads.always", "metastore.task.threads.always", + EventCleanerTask.class.getName() + "," + RuntimeStatsCleanerTask.class.getName() + "," + + "org.apache.hadoop.hive.metastore.repl.DumpDirCleanerTask" + "," + + "org.apache.hadoop.hive.metastore.HiveProtoEventsCleanerTask", + "Comma separated list of tasks that will be started in separate threads. These will " + + "always be started, regardless of whether the metastore is running in embedded mode " + + "or in server mode. They must implement " + MetastoreTaskThread.class.getName()), + TASK_THREADS_REMOTE_ONLY("metastore.task.threads.remote", "metastore.task.threads.remote", + AcidHouseKeeperService.class.getName() + "," + + AcidOpenTxnsCounterService.class.getName() + "," + + AcidCompactionHistoryService.class.getName() + "," + + AcidWriteSetService.class.getName() + "," + + MaterializationsRebuildLockCleanerTask.class.getName(), + "Command separated list of tasks that will be started in separate threads. These will be" + + " started only when the metastore is running as a separate service. They must " + + "implement " + MetastoreTaskThread.class.getName()), + STORAGE_SCHEMA_READER_IMPL("metastore.storage.schema.reader.impl", "metastore.storage.schema.reader.impl", + DefaultStorageSchemaReader.class.getName(), + "The class to use to read schemas from storage. It must implement " + + "org.apache.hadoop.hive.metastore.StorageSchemaReader"); + + private final String varname; + private final String hiveName; + private final Object defaultVal; + private final Validator validator; + private final boolean caseSensitive; + private final String description; + + ConfVars(String varname, String hiveName, String defaultVal, String description) { + this.varname = varname; + this.hiveName = hiveName; + this.defaultVal = defaultVal; + validator = null; + caseSensitive = false; + this.description = description; + } + + public Object getDefaultVal() { + return defaultVal; + } + } + + /** + * Get the variable as a string + * @param conf configuration to retrieve it from + * @param var variable to retrieve + * @return value, or default value if value not in config file + */ + public static String getVar(Configuration conf, ConfVars var) { + assert var.defaultVal.getClass() == String.class; + String val = conf.get(var.varname); + return val == null ? conf.get(var.hiveName, (String)var.defaultVal) : val; + } + + /** + * Get the variable as a string + * @param conf configuration to retrieve it from + * @param var variable to retrieve + * @param defaultVal default to return if the variable is unset + * @return value, or default value passed in if the value is not in the config file + */ + public static String getVar(Configuration conf, ConfVars var, String defaultVal) { + assert var.defaultVal.getClass() == String.class; + String val = conf.get(var.varname); + return val == null ? conf.get(var.hiveName, defaultVal) : val; + } + + /** + * Set the variable as a string + * @param conf configuration file to set it in + * @param var variable to set + * @param val value to set it to + */ + public static void setVar(Configuration conf, ConfVars var, String val) { + assert var.defaultVal.getClass() == String.class; + conf.set(var.varname, val); + } + + /** + * Treat a configuration value as a comma separated list. + * @param conf configuration to retrieve it from + * @param var variable to retrieve + * @return collection of strings. If the value is unset it will return an empty collection. + */ + public static Collection getStringCollection(Configuration conf, ConfVars var) { + assert var.defaultVal.getClass() == String.class; + String val = conf.get(var.varname); + if (val == null) { + val = conf.get(var.hiveName, (String)var.defaultVal); + } + if (val == null) { + return Collections.emptySet(); + } + return StringUtils.asSet(val.split(",")); + } + +} diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/security/DelegationTokenTool.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/security/DelegationTokenTool.java index 3637db09f1..75eac35a04 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/security/DelegationTokenTool.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/security/DelegationTokenTool.java @@ -30,6 +30,7 @@ import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.conf.MetastoreServerConf; import org.apache.hadoop.util.GenericOptionsParser; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.Tool; @@ -193,7 +194,8 @@ private void init() throws Exception { conf.addResource(new Path(confLocation)); String tokenStoreClassName = - MetastoreConf.getVar(conf,MetastoreConf.ConfVars.DELEGATION_TOKEN_STORE_CLS, ""); + MetastoreServerConf.getVar(conf, MetastoreServerConf.ConfVars.DELEGATION_TOKEN_STORE_CLS, + ""); if (StringUtils.isBlank(tokenStoreClassName)) { throw new Exception("Could not find Delegation TokenStore implementation."); } diff --git a/standalone-metastore/metastore-common/src/test/java/org/apache/hadoop/hive/metastore/MetaStoreTestUtils.java b/standalone-metastore/metastore-common/src/test/java/org/apache/hadoop/hive/metastore/MetaStoreTestUtils.java index 3d36b60ec9..9bbf368497 100644 --- a/standalone-metastore/metastore-common/src/test/java/org/apache/hadoop/hive/metastore/MetaStoreTestUtils.java +++ b/standalone-metastore/metastore-common/src/test/java/org/apache/hadoop/hive/metastore/MetaStoreTestUtils.java @@ -33,6 +33,7 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; +import org.apache.hadoop.hive.metastore.conf.MetastoreServerConf; import org.apache.hadoop.hive.metastore.events.EventCleanerTask; import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge; import org.apache.thrift.TException; @@ -253,9 +254,9 @@ public static int findFreePortExcepting(int portToExclude) throws IOException { * @param conf Configuration object */ public static void setConfForStandloneMode(Configuration conf) { - if (MetastoreConf.getVar(conf, ConfVars.TASK_THREADS_ALWAYS).equals( - ConfVars.TASK_THREADS_ALWAYS.getDefaultVal())) { - MetastoreConf.setVar(conf, ConfVars.TASK_THREADS_ALWAYS, + if (MetastoreServerConf.getVar(conf, MetastoreServerConf.ConfVars.TASK_THREADS_ALWAYS).equals( + MetastoreServerConf.ConfVars.TASK_THREADS_ALWAYS.getDefaultVal())) { + MetastoreServerConf.setVar(conf, MetastoreServerConf.ConfVars.TASK_THREADS_ALWAYS, EventCleanerTask.class.getName()); } if (MetastoreConf.getVar(conf, ConfVars.EXPRESSION_PROXY_CLASS).equals( diff --git a/standalone-metastore/metastore-common/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java b/standalone-metastore/metastore-common/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java index cb32236d54..206bce970a 100644 --- a/standalone-metastore/metastore-common/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java +++ b/standalone-metastore/metastore-common/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java @@ -45,6 +45,7 @@ import org.apache.hadoop.hive.metastore.client.builder.TableBuilder; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; +import org.apache.hadoop.hive.metastore.conf.MetastoreServerConf; import org.apache.hadoop.hive.metastore.utils.FileUtils; import org.apache.hadoop.hive.metastore.utils.SecurityUtils; import org.datanucleus.api.jdo.JDOPersistenceManager; @@ -126,7 +127,8 @@ public void setUp() throws Exception { MetaStoreTestUtils.setConfForStandloneMode(conf); MetastoreConf.setLongVar(conf, ConfVars.BATCH_RETRIEVE_MAX, 2); MetastoreConf.setLongVar(conf, ConfVars.LIMIT_PARTITION_REQUEST, DEFAULT_LIMIT_PARTITION_REQUEST); - MetastoreConf.setVar(conf, ConfVars.STORAGE_SCHEMA_READER_IMPL, "no.such.class"); + MetastoreServerConf.setVar(conf, MetastoreServerConf.ConfVars.STORAGE_SCHEMA_READER_IMPL, + "no.such.class"); } @Test -- 2.16.3