From ad4ec339d214d8bb552cc12ece2af0d869e6d59a Mon Sep 17 00:00:00 2001 From: Alexander Kolbasov Date: Thu, 9 Aug 2018 16:49:23 -0700 Subject: [PATCH 1/1] HIVE-20196: Remove MetastoreConf dependency on server-specific classes --- .../hadoop/hive/metastore/conf/MetastoreConf.java | 56 +++++++++++++--------- 1 file changed, 34 insertions(+), 22 deletions(-) diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java index cf5fbbefa6..c23fe4d744 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java @@ -19,17 +19,6 @@ import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.metastore.DefaultStorageSchemaReader; -import org.apache.hadoop.hive.metastore.HiveAlterHandler; -import org.apache.hadoop.hive.metastore.MaterializationsRebuildLockCleanerTask; -import org.apache.hadoop.hive.metastore.MetastoreTaskThread; -import org.apache.hadoop.hive.metastore.RuntimeStatsCleanerTask; -import org.apache.hadoop.hive.metastore.events.EventCleanerTask; -import org.apache.hadoop.hive.metastore.security.MetastoreDelegationTokenManager; -import org.apache.hadoop.hive.metastore.txn.AcidCompactionHistoryService; -import org.apache.hadoop.hive.metastore.txn.AcidHouseKeeperService; -import org.apache.hadoop.hive.metastore.txn.AcidOpenTxnsCounterService; -import org.apache.hadoop.hive.metastore.txn.AcidWriteSetService; import org.apache.hadoop.hive.metastore.utils.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -70,6 +59,29 @@ private static final Logger LOG = LoggerFactory.getLogger(MetastoreConf.class); private static final Pattern TIME_UNIT_SUFFIX = Pattern.compile("([0-9]+)([a-zA-Z]+)"); + private static final String DEFAULT_STORAGE_SCHEMA_READER_CLASS = + "org.apache.hadoop.hive.metastore.DefaultStorageSchemaReader"; + private static final String HIVE_ALTER_HANDLE_CLASS = + "org.apache.hadoop.hive.metastore.HiveAlterHandler"; + private static final String MATERIALZIATIONS_REBUILD_LOCK_CLEANER_TASK_CLASS = + "org.apache.hadoop.hive.metastore.MaterializationsRebuildLockCleanerTask"; + private static final String METASTORE_TASK_THREAD_CLASS = + "org.apache.hadoop.hive.metastore.MetastoreTaskThread"; + private static final String RUNTIME_STATS_CLEANER_TASK_CLASS = + "org.apache.hadoop.hive.metastore.RuntimeStatsCleanerTask"; + private static final String EVENT_CLEANER_TASK_CLASS = + "org.apache.hadoop.hive.metastore.events.EventCleanerTask"; + private static final String METASTORE_DELEGATION_MANAGER_CLASS = + "org.apache.hadoop.hive.metastore.security.MetastoreDelegationTokenManager"; + private static final String ACID_COMPACTION_HISTORY_SERVICE_CLASS = + "org.apache.hadoop.hive.metastore.txn.AcidCompactionHistoryService"; + private static final String ACID_HOUSE_KEEPER_SERVICE_CLASS = + "org.apache.hadoop.hive.metastore.txn.AcidHouseKeeperService"; + private static final String ACID_OPEN_TXNS_COUNTER_SERVICE_CLASS = + "org.apache.hadoop.hive.metastore.txn.AcidOpenTxnsCounterService"; + private static final String ACID_WRITE_SET_SERVICE_CLASS = + "org.apache.hadoop.hive.metastore.txn.AcidWriteSetService"; + private static final Map metaConfs = new HashMap<>(); private static URL hiveDefaultURL = null; private static URL hiveSiteURL = null; @@ -260,7 +272,7 @@ public static ConfVars getMetaConf(String name) { "hive.metastore.aggregate.stats.cache.ttl", 600, TimeUnit.SECONDS, "Number of seconds for a cached node to be active in the cache before they become stale."), ALTER_HANDLER("metastore.alter.handler", "hive.metastore.alter.impl", - HiveAlterHandler.class.getName(), + HIVE_ALTER_HANDLE_CLASS, "Alter handler. For now defaults to the Hive one. Really need a better default option"), ASYNC_LOG_ENABLED("metastore.async.log.enabled", "hive.async.log.enabled", true, "Whether to enable Log4j2's asynchronous logging. Asynchronous logging can give\n" + @@ -436,7 +448,7 @@ public static ConfVars getMetaConf(String name) { DELEGATION_TOKEN_RENEW_INTERVAL("metastore.cluster.delegation.token.renew-interval", "hive.cluster.delegation.token.renew-interval", 1, TimeUnit.DAYS, ""), DELEGATION_TOKEN_STORE_CLS("metastore.cluster.delegation.token.store.class", - "hive.cluster.delegation.token.store.class", MetastoreDelegationTokenManager.class.getName(), + "hive.cluster.delegation.token.store.class", METASTORE_DELEGATION_MANAGER_CLASS, "Class to store delegation tokens"), DETACH_ALL_ON_COMMIT("javax.jdo.option.DetachAllOnCommit", "javax.jdo.option.DetachAllOnCommit", true, @@ -747,7 +759,7 @@ public static ConfVars getMetaConf(String name) { "hive.metastore.stats.auto.analyze.worker.count", 1, "Number of parallel analyze commands to run for background stats update."), STORAGE_SCHEMA_READER_IMPL("metastore.storage.schema.reader.impl", "metastore.storage.schema.reader.impl", - DefaultStorageSchemaReader.class.getName(), + DEFAULT_STORAGE_SCHEMA_READER_CLASS, "The class to use to read schemas from storage. It must implement " + "org.apache.hadoop.hive.metastore.StorageSchemaReader"), STORE_MANAGER_TYPE("datanucleus.storeManagerType", "datanucleus.storeManagerType", "rdbms", "metadata store type"), @@ -761,21 +773,21 @@ public static ConfVars getMetaConf(String name) { + "The only supported special character right now is '/'. This flag applies only to quoted table names.\n" + "The default value is true."), TASK_THREADS_ALWAYS("metastore.task.threads.always", "metastore.task.threads.always", - EventCleanerTask.class.getName() + "," + RuntimeStatsCleanerTask.class.getName() + "," + + EVENT_CLEANER_TASK_CLASS + "," + RUNTIME_STATS_CLEANER_TASK_CLASS + "," + "org.apache.hadoop.hive.metastore.repl.DumpDirCleanerTask" + "," + "org.apache.hadoop.hive.metastore.HiveProtoEventsCleanerTask", "Comma separated list of tasks that will be started in separate threads. These will " + "always be started, regardless of whether the metastore is running in embedded mode " + - "or in server mode. They must implement " + MetastoreTaskThread.class.getName()), + "or in server mode. They must implement " + METASTORE_TASK_THREAD_CLASS), TASK_THREADS_REMOTE_ONLY("metastore.task.threads.remote", "metastore.task.threads.remote", - AcidHouseKeeperService.class.getName() + "," + - AcidOpenTxnsCounterService.class.getName() + "," + - AcidCompactionHistoryService.class.getName() + "," + - AcidWriteSetService.class.getName() + "," + - MaterializationsRebuildLockCleanerTask.class.getName(), + ACID_HOUSE_KEEPER_SERVICE_CLASS + "," + + ACID_OPEN_TXNS_COUNTER_SERVICE_CLASS + "," + + ACID_COMPACTION_HISTORY_SERVICE_CLASS + "," + + ACID_WRITE_SET_SERVICE_CLASS + "," + + MATERIALZIATIONS_REBUILD_LOCK_CLEANER_TASK_CLASS, "Command separated list of tasks that will be started in separate threads. These will be" + " started only when the metastore is running as a separate service. They must " + - "implement " + MetastoreTaskThread.class.getName()), + "implement " + METASTORE_TASK_THREAD_CLASS), TCP_KEEP_ALIVE("metastore.server.tcp.keepalive", "hive.metastore.server.tcp.keepalive", true, "Whether to enable TCP keepalive for the metastore server. Keepalive will prevent accumulation of half-open connections."), -- 2.16.3