diff --git .gitignore .gitignore index 89169b0a36..d56f31fd8d 100644 --- .gitignore +++ .gitignore @@ -30,3 +30,4 @@ conf/hive-default.xml.template itests/hive-blobstore/src/test/resources/blobstore-conf.xml .DS_Store patchprocess +standalone-metastore/src/gen/org/apache/hadoop/hive/metastore/annotation/package-info.java diff --git beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java index 4d7373a17a..84963af10e 100644 --- beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java +++ beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java @@ -38,6 +38,7 @@ import org.apache.hadoop.hive.metastore.MetaStoreSchemaInfoFactory; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper; import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper.MetaStoreConnectionInfo; import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper.NestedScriptParser; @@ -1016,8 +1017,8 @@ String buildToLog() throws IOException { private String[] argsWith(String password) throws IOException { return new String[] { - "-u", url == null ? HiveSchemaHelper.getValidConfVar(ConfVars.METASTORECONNECTURLKEY, hiveConf) : url, - "-d", driver == null ? HiveSchemaHelper.getValidConfVar(ConfVars.METASTORE_CONNECTION_DRIVER, hiveConf) : driver, + "-u", url == null ? HiveSchemaHelper.getValidConfVar(MetastoreConf.ConfVars.CONNECTURLKEY, hiveConf) : url, + "-d", driver == null ? HiveSchemaHelper.getValidConfVar(MetastoreConf.ConfVars.CONNECTION_DRIVER, hiveConf) : driver, "-n", userName, "-p", password, "-f", sqlScriptFile diff --git beeline/src/test/org/apache/hive/beeline/TestHiveSchemaTool.java beeline/src/test/org/apache/hive/beeline/TestHiveSchemaTool.java index d9b8f423ae..03c9f7c24c 100644 --- beeline/src/test/org/apache/hive/beeline/TestHiveSchemaTool.java +++ beeline/src/test/org/apache/hive/beeline/TestHiveSchemaTool.java @@ -18,6 +18,7 @@ package org.apache.hive.beeline; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper; import org.junit.After; import org.junit.Before; @@ -55,10 +56,10 @@ public void setup() throws IOException { mockStatic(HiveSchemaHelper.class); when(HiveSchemaHelper - .getValidConfVar(eq(HiveConf.ConfVars.METASTORECONNECTURLKEY), same(hiveConf))) + .getValidConfVar(eq(MetastoreConf.ConfVars.CONNECTURLKEY), same(hiveConf))) .thenReturn("someURL"); when(HiveSchemaHelper - .getValidConfVar(eq(HiveConf.ConfVars.METASTORE_CONNECTION_DRIVER), same(hiveConf))) + .getValidConfVar(eq(MetastoreConf.ConfVars.CONNECTION_DRIVER), same(hiveConf))) .thenReturn("someDriver"); File file = new File(scriptFile); @@ -71,9 +72,9 @@ public void setup() throws IOException { @After public void globalAssert() throws IOException { verifyStatic(); - HiveSchemaHelper.getValidConfVar(eq(HiveConf.ConfVars.METASTORECONNECTURLKEY), same(hiveConf)); + HiveSchemaHelper.getValidConfVar(eq(MetastoreConf.ConfVars.CONNECTURLKEY), same(hiveConf)); HiveSchemaHelper - .getValidConfVar(eq(HiveConf.ConfVars.METASTORE_CONNECTION_DRIVER), same(hiveConf)); + .getValidConfVar(eq(MetastoreConf.ConfVars.CONNECTION_DRIVER), same(hiveConf)); new File(scriptFile).delete(); } diff --git hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java index 49520eff49..1e73a4b9ba 100644 --- hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java +++ hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java @@ -414,7 +414,7 @@ public static void runWorker(HiveConf hiveConf) throws MetaException { AtomicBoolean stop = new AtomicBoolean(true); Worker t = new Worker(); t.setThreadId((int) t.getId()); - t.setHiveConf(hiveConf); + t.setConf(hiveConf); AtomicBoolean looped = new AtomicBoolean(); t.init(stop, looped); t.run(); diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java index fc85d86c41..69efc98024 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java @@ -24,6 +24,7 @@ import java.util.ArrayList; import java.util.List; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.UtilsForTest; import org.apache.hadoop.hive.cli.CliSessionState; import org.apache.hadoop.hive.conf.HiveConf; @@ -49,7 +50,7 @@ public static class DummyMetaStoreFilterHookImpl extends DefaultMetaStoreFilterHookImpl { public static boolean blockResults = false; - public DummyMetaStoreFilterHookImpl(HiveConf conf) { + public DummyMetaStoreFilterHookImpl(Configuration conf) { super(conf); } diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java index 44b244b63e..4bfc5be397 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java @@ -200,7 +200,7 @@ public void schemaEvolutionAddColDynamicPartitioningInsert() throws Exception { Initiator initiator = new Initiator(); initiator.setThreadId((int)initiator.getId()); conf.setIntVar(HiveConf.ConfVars.HIVE_COMPACTOR_DELTA_NUM_THRESHOLD, 0); - initiator.setHiveConf(conf); + initiator.setConf(conf); AtomicBoolean stop = new AtomicBoolean(); stop.set(true); initiator.init(stop, new AtomicBoolean()); @@ -303,7 +303,7 @@ public void schemaEvolutionAddColDynamicPartitioningUpdate() throws Exception { initiator.setThreadId((int)initiator.getId()); // Set to 1 so insert doesn't set it off but update does conf.setIntVar(HiveConf.ConfVars.HIVE_COMPACTOR_DELTA_NUM_THRESHOLD, 1); - initiator.setHiveConf(conf); + initiator.setConf(conf); AtomicBoolean stop = new AtomicBoolean(); stop.set(true); initiator.init(stop, new AtomicBoolean()); @@ -455,7 +455,7 @@ public void testStatsAfterCompactionPartTbl() throws Exception { txnHandler.compact(rqst); Worker t = new Worker(); t.setThreadId((int) t.getId()); - t.setHiveConf(conf); + t.setConf(conf); AtomicBoolean stop = new AtomicBoolean(); AtomicBoolean looped = new AtomicBoolean(); stop.set(true); @@ -511,7 +511,7 @@ public void dynamicPartitioningInsert() throws Exception { Initiator initiator = new Initiator(); initiator.setThreadId((int)initiator.getId()); conf.setIntVar(HiveConf.ConfVars.HIVE_COMPACTOR_DELTA_NUM_THRESHOLD, 0); - initiator.setHiveConf(conf); + initiator.setConf(conf); AtomicBoolean stop = new AtomicBoolean(); stop.set(true); initiator.init(stop, new AtomicBoolean()); @@ -551,7 +551,7 @@ public void dynamicPartitioningUpdate() throws Exception { initiator.setThreadId((int)initiator.getId()); // Set to 1 so insert doesn't set it off but update does conf.setIntVar(HiveConf.ConfVars.HIVE_COMPACTOR_DELTA_NUM_THRESHOLD, 1); - initiator.setHiveConf(conf); + initiator.setConf(conf); AtomicBoolean stop = new AtomicBoolean(); stop.set(true); initiator.init(stop, new AtomicBoolean()); @@ -593,7 +593,7 @@ public void dynamicPartitioningDelete() throws Exception { initiator.setThreadId((int)initiator.getId()); // Set to 2 so insert and update don't set it off but delete does conf.setIntVar(HiveConf.ConfVars.HIVE_COMPACTOR_DELTA_NUM_THRESHOLD, 2); - initiator.setHiveConf(conf); + initiator.setConf(conf); AtomicBoolean stop = new AtomicBoolean(); stop.set(true); initiator.init(stop, new AtomicBoolean()); @@ -643,7 +643,7 @@ public void minorCompactWhileStreaming() throws Exception { txnHandler.compact(new CompactionRequest(dbName, tblName, CompactionType.MINOR)); Worker t = new Worker(); t.setThreadId((int) t.getId()); - t.setHiveConf(conf); + t.setConf(conf); AtomicBoolean stop = new AtomicBoolean(true); AtomicBoolean looped = new AtomicBoolean(); t.init(stop, looped); @@ -705,7 +705,7 @@ public void majorCompactWhileStreaming() throws Exception { txnHandler.compact(new CompactionRequest(dbName, tblName, CompactionType.MAJOR)); Worker t = new Worker(); t.setThreadId((int) t.getId()); - t.setHiveConf(conf); + t.setConf(conf); AtomicBoolean stop = new AtomicBoolean(true); AtomicBoolean looped = new AtomicBoolean(); t.init(stop, looped); @@ -762,7 +762,7 @@ public void minorCompactAfterAbort() throws Exception { txnHandler.compact(new CompactionRequest(dbName, tblName, CompactionType.MINOR)); Worker t = new Worker(); t.setThreadId((int) t.getId()); - t.setHiveConf(conf); + t.setConf(conf); AtomicBoolean stop = new AtomicBoolean(true); AtomicBoolean looped = new AtomicBoolean(); t.init(stop, looped); @@ -828,7 +828,7 @@ public void majorCompactAfterAbort() throws Exception { txnHandler.compact(new CompactionRequest(dbName, tblName, CompactionType.MAJOR)); Worker t = new Worker(); t.setThreadId((int) t.getId()); - t.setHiveConf(conf); + t.setConf(conf); AtomicBoolean stop = new AtomicBoolean(true); AtomicBoolean looped = new AtomicBoolean(); t.init(stop, looped); @@ -886,7 +886,7 @@ public void majorCompactWhileStreamingForSplitUpdate() throws Exception { txnHandler.compact(new CompactionRequest(dbName, tblName, CompactionType.MAJOR)); Worker t = new Worker(); t.setThreadId((int) t.getId()); - t.setHiveConf(conf); + t.setConf(conf); AtomicBoolean stop = new AtomicBoolean(true); AtomicBoolean looped = new AtomicBoolean(); t.init(stop, looped); @@ -939,7 +939,7 @@ public void testMinorCompactionForSplitUpdateWithInsertsAndDeletes() throws Exce txnHandler.compact(new CompactionRequest(dbName, tblName, CompactionType.MINOR)); Worker t = new Worker(); t.setThreadId((int) t.getId()); - t.setHiveConf(conf); + t.setConf(conf); AtomicBoolean stop = new AtomicBoolean(true); AtomicBoolean looped = new AtomicBoolean(); t.init(stop, looped); @@ -1016,7 +1016,7 @@ public void testMinorCompactionForSplitUpdateWithOnlyInserts() throws Exception txnHandler.compact(new CompactionRequest(dbName, tblName, CompactionType.MINOR)); Worker t = new Worker(); t.setThreadId((int) t.getId()); - t.setHiveConf(conf); + t.setConf(conf); AtomicBoolean stop = new AtomicBoolean(true); AtomicBoolean looped = new AtomicBoolean(); t.init(stop, looped); @@ -1095,7 +1095,7 @@ public void minorCompactWhileStreamingWithSplitUpdate() throws Exception { txnHandler.compact(new CompactionRequest(dbName, tblName, CompactionType.MINOR)); Worker t = new Worker(); t.setThreadId((int) t.getId()); - t.setHiveConf(conf); + t.setConf(conf); AtomicBoolean stop = new AtomicBoolean(true); AtomicBoolean looped = new AtomicBoolean(); t.init(stop, looped); @@ -1203,7 +1203,7 @@ public void testTableProperties() throws Exception { AtomicBoolean stop = new AtomicBoolean(true); Worker t = new Worker(); t.setThreadId((int) t.getId()); - t.setHiveConf(conf); + t.setConf(conf); AtomicBoolean looped = new AtomicBoolean(); t.init(stop, looped); t.run(); @@ -1213,7 +1213,7 @@ public void testTableProperties() throws Exception { stop = new AtomicBoolean(true); t = new Worker(); t.setThreadId((int) t.getId()); - t.setHiveConf(conf); + t.setConf(conf); looped = new AtomicBoolean(); t.init(stop, looped); t.run(); @@ -1265,7 +1265,7 @@ public void testTableProperties() throws Exception { stop = new AtomicBoolean(true); t = new Worker(); t.setThreadId((int) t.getId()); - t.setHiveConf(conf); + t.setConf(conf); looped = new AtomicBoolean(); t.init(stop, looped); t.run(); @@ -1416,7 +1416,7 @@ static void runInitiator(HiveConf hiveConf) throws MetaException { AtomicBoolean stop = new AtomicBoolean(true); Initiator t = new Initiator(); t.setThreadId((int) t.getId()); - t.setHiveConf(hiveConf); + t.setConf(hiveConf); AtomicBoolean looped = new AtomicBoolean(); t.init(stop, looped); t.run(); @@ -1426,7 +1426,7 @@ static void runWorker(HiveConf hiveConf) throws MetaException { AtomicBoolean stop = new AtomicBoolean(true); Worker t = new Worker(); t.setThreadId((int) t.getId()); - t.setHiveConf(hiveConf); + t.setConf(hiveConf); AtomicBoolean looped = new AtomicBoolean(); t.init(stop, looped); t.run(); @@ -1436,7 +1436,7 @@ static void runCleaner(HiveConf hiveConf) throws MetaException { AtomicBoolean stop = new AtomicBoolean(true); Cleaner t = new Cleaner(); t.setThreadId((int) t.getId()); - t.setHiveConf(hiveConf); + t.setConf(hiveConf); AtomicBoolean looped = new AtomicBoolean(); t.init(stop, looped); t.run(); diff --git metastore/pom.xml metastore/pom.xml index 62bddd15ec..8c356499d2 100644 --- metastore/pom.xml +++ metastore/pom.xml @@ -61,11 +61,6 @@ ${guava.version} - com.google.protobuf - protobuf-java - ${protobuf.version} - - org.apache.hbase hbase-client ${hbase.version} @@ -265,39 +260,6 @@ - - protobuf - - - - org.apache.maven.plugins - maven-antrun-plugin - - - generate-protobuf-sources - generate-sources - - - - - Building HBase Metastore Protobuf - - - - - - - - - - run - - - - - - - @@ -343,7 +305,6 @@ src/model - src/gen/protobuf/gen-java diff --git metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 5812a1bf4f..f559e243f1 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -7900,7 +7900,7 @@ private static MetaStoreThread instantiateThread(String classname) throws Except private static void initializeAndStartThread(MetaStoreThread thread, HiveConf conf) throws MetaException { LOG.info("Starting metastore thread of type " + thread.getClass().getName()); - thread.setHiveConf(conf); + thread.setConf(conf); thread.setThreadId(nextThreadId++); thread.init(new AtomicBoolean(), new AtomicBoolean()); thread.start(); diff --git metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java index 70451c4e62..70b23fe98c 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java @@ -49,6 +49,7 @@ import javax.security.auth.login.LoginException; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.common.ObjectPair; import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.common.ValidTxnList; @@ -263,7 +264,7 @@ private MetaStoreFilterHook loadFilterHooks() throws IllegalStateException { String msg = "Unable to create instance of " + authProviderClass.getName() + ": "; try { Constructor constructor = - authProviderClass.getConstructor(HiveConf.class); + authProviderClass.getConstructor(Configuration.class); return constructor.newInstance(conf); } catch (NoSuchMethodException e) { throw new IllegalStateException(msg + e.getMessage(), e); diff --git metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java index b122c431b2..634ce1a5f8 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java @@ -1301,7 +1301,7 @@ public AggrStats aggrColStatsForPartitions(String dbName, String tableName, AggrColStats colStatsAggrCached; List colStatsAggrFromDB; int maxPartsPerCacheNode = aggrStatsCache.getMaxPartsPerCacheNode(); - float fpp = aggrStatsCache.getFalsePositiveProbability(); + double fpp = aggrStatsCache.getFalsePositiveProbability(); colStatsList = new ArrayList(); // Bloom filter for the new node that we will eventually add to the cache BloomFilter bloomFilter = createPartsBloomFilter(maxPartsPerCacheNode, fpp, partNames); @@ -1343,7 +1343,7 @@ public AggrStats aggrColStatsForPartitions(String dbName, String tableName, return new AggrStats(colStatsList, partsFound); } - private BloomFilter createPartsBloomFilter(int maxPartsPerCacheNode, float fpp, + private BloomFilter createPartsBloomFilter(int maxPartsPerCacheNode, double fpp, List partNames) { BloomFilter bloomFilter = new BloomFilter(maxPartsPerCacheNode, fpp); for (String partName : partNames) { diff --git ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/AuthorizationMetaStoreFilterHook.java ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/AuthorizationMetaStoreFilterHook.java index b08c63dc6c..96f1b9ef22 100644 --- ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/AuthorizationMetaStoreFilterHook.java +++ ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/AuthorizationMetaStoreFilterHook.java @@ -20,10 +20,10 @@ import java.util.ArrayList; import java.util.List; +import org.apache.hadoop.conf.Configuration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.hive.common.classification.InterfaceAudience.Private; -import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.DefaultMetaStoreFilterHookImpl; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject.HivePrivilegeObjectType; @@ -38,7 +38,7 @@ public static final Logger LOG = LoggerFactory.getLogger(AuthorizationMetaStoreFilterHook.class); - public AuthorizationMetaStoreFilterHook(HiveConf conf) { + public AuthorizationMetaStoreFilterHook(Configuration conf) { super(conf); } diff --git ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java index 6adfaced30..f9afd76736 100644 --- ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java +++ ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java @@ -65,7 +65,7 @@ @Override public void run() { if (cleanerCheckInterval == 0) { - cleanerCheckInterval = conf.getTimeVar( + cleanerCheckInterval = hiveConf.getTimeVar( HiveConf.ConfVars.HIVE_COMPACTOR_CLEANER_RUN_INTERVAL, TimeUnit.MILLISECONDS); } diff --git ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java index 4d6e24ef38..c8f4df72d9 100644 --- ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java +++ ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hive.ql.txn.compactor; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -28,6 +29,7 @@ import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.txn.CompactionInfo; import org.apache.hadoop.hive.metastore.txn.TxnStore; import org.apache.hadoop.hive.metastore.txn.TxnUtils; @@ -50,7 +52,8 @@ static final private String CLASS_NAME = CompactorThread.class.getName(); static final private Logger LOG = LoggerFactory.getLogger(CLASS_NAME); - protected HiveConf conf; + protected Configuration conf; + protected HiveConf hiveConf; protected TxnStore txnHandler; protected RawStore rs; protected int threadId; @@ -58,8 +61,17 @@ protected AtomicBoolean looped; @Override - public void setHiveConf(HiveConf conf) { - this.conf = conf; + public void setConf(Configuration configuration) { + conf = configuration; + // TODO for now, keep a copy of HiveConf around as we need to call other methods with it. + // This should be removed once everything that this calls that requires HiveConf is moved to + // the standalone metastore. + hiveConf = (conf instanceof HiveConf) ? (HiveConf)conf : new HiveConf(conf, HiveConf.class); + } + + @Override + public Configuration getConf() { + return conf; } @Override @@ -76,11 +88,11 @@ public void init(AtomicBoolean stop, AtomicBoolean looped) throws MetaException setDaemon(true); // this means the process will exit without waiting for this thread // Get our own instance of the transaction handler - txnHandler = TxnUtils.getTxnStore(conf); + txnHandler = TxnUtils.getTxnStore(hiveConf); // Get our own connection to the database so we can get table and partition information. - rs = RawStoreProxy.getProxy(conf, conf, - conf.getVar(HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL), threadId); + rs = RawStoreProxy.getProxy(hiveConf, conf, + MetastoreConf.getVar(conf, MetastoreConf.ConfVars.RAW_STORE_IMPL), threadId); } /** @@ -163,7 +175,7 @@ protected String findUserToRunAs(String location, Table t) throws IOException, LOG.debug("Unable to stat file as current user, trying as table owner"); // Now, try it as the table owner and see if we get better luck. - final List wrapper = new ArrayList(1); + final List wrapper = new ArrayList<>(1); UserGroupInformation ugi = UserGroupInformation.createProxyUser(t.getOwner(), UserGroupInformation.getLoginUser()); ugi.doAs(new PrivilegedExceptionAction() { diff --git ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java index af4a1da4dc..4920fc582a 100644 --- ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java +++ ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java @@ -188,7 +188,7 @@ public void run() { public void init(AtomicBoolean stop, AtomicBoolean looped) throws MetaException { super.init(stop, looped); checkInterval = - conf.getTimeVar(HiveConf.ConfVars.HIVE_COMPACTOR_CHECK_INTERVAL, TimeUnit.MILLISECONDS) ; + hiveConf.getTimeVar(HiveConf.ConfVars.HIVE_COMPACTOR_CHECK_INTERVAL, TimeUnit.MILLISECONDS) ; } private void recoverFailedCompactions(boolean remoteOnly) throws MetaException { diff --git ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java index 2d6cce9410..6d389c980a 100644 --- ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java +++ ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java @@ -157,20 +157,20 @@ public void run() { LOG.info("Starting " + ci.type.toString() + " compaction for " + ci.getFullPartitionName()); - final StatsUpdater su = StatsUpdater.init(ci, txnHandler.findColumnsWithStats(ci), conf, + final StatsUpdater su = StatsUpdater.init(ci, txnHandler.findColumnsWithStats(ci), hiveConf, runJobAsSelf(runAs) ? runAs : t.getOwner()); final CompactorMR mr = new CompactorMR(); launchedJob = true; try { if (runJobAsSelf(runAs)) { - mr.run(conf, jobName.toString(), t, sd, txns, ci, su, txnHandler); + mr.run(hiveConf, jobName.toString(), t, sd, txns, ci, su, txnHandler); } else { UserGroupInformation ugi = UserGroupInformation.createProxyUser(t.getOwner(), UserGroupInformation.getLoginUser()); ugi.doAs(new PrivilegedExceptionAction() { @Override public Object run() throws Exception { - mr.run(conf, jobName.toString(), t, sd, txns, ci, su, txnHandler); + mr.run(hiveConf, jobName.toString(), t, sd, txns, ci, su, txnHandler); return null; } }); @@ -182,7 +182,7 @@ public Object run() throws Exception { } } txnHandler.markCompacted(ci); - if (conf.getBoolVar(HiveConf.ConfVars.HIVE_IN_TEST)) { + if (hiveConf.getBoolVar(HiveConf.ConfVars.HIVE_IN_TEST)) { mrJob = mr.getMrJob(); } } catch (Exception e) { diff --git ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java index 21b4a2ce55..d08371d89c 100644 --- ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java +++ ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java @@ -228,7 +228,7 @@ private void testOrcPPD(boolean enablePPD) throws Exception { runStatementOnDriver("alter table " + Table.ACIDTBL + " compact 'MAJOR'"); Worker t = new Worker(); t.setThreadId((int) t.getId()); - t.setHiveConf(hiveConf); + t.setConf(hiveConf); AtomicBoolean stop = new AtomicBoolean(); AtomicBoolean looped = new AtomicBoolean(); stop.set(true); @@ -275,7 +275,7 @@ public void testAlterTable() throws Exception { runStatementOnDriver("alter table "+ Table.ACIDTBL + " compact 'MAJOR'"); Worker t = new Worker(); t.setThreadId((int) t.getId()); - t.setHiveConf(hiveConf); + t.setConf(hiveConf); AtomicBoolean stop = new AtomicBoolean(); AtomicBoolean looped = new AtomicBoolean(); stop.set(true); @@ -1010,7 +1010,7 @@ void testInitiatorWithMultipleFailedCompactionsForVariousTblProperties(String tb //this should not schedule a new compaction due to prior failures, but will create Attempted entry Initiator init = new Initiator(); init.setThreadId((int)init.getId()); - init.setHiveConf(hiveConf); + init.setConf(hiveConf); init.init(stop, new AtomicBoolean()); init.run(); int numAttemptedCompactions = 1; @@ -1160,7 +1160,7 @@ public static void runWorker(HiveConf hiveConf) throws MetaException { AtomicBoolean stop = new AtomicBoolean(true); Worker t = new Worker(); t.setThreadId((int) t.getId()); - t.setHiveConf(hiveConf); + t.setConf(hiveConf); AtomicBoolean looped = new AtomicBoolean(); t.init(stop, looped); t.run(); @@ -1169,7 +1169,7 @@ public static void runCleaner(HiveConf hiveConf) throws MetaException { AtomicBoolean stop = new AtomicBoolean(true); Cleaner t = new Cleaner(); t.setThreadId((int) t.getId()); - t.setHiveConf(hiveConf); + t.setConf(hiveConf); AtomicBoolean looped = new AtomicBoolean(); t.init(stop, looped); t.run(); @@ -1220,7 +1220,7 @@ protected void writeBetweenWorkerAndCleanerForVariousTblProperties(String tblPro txnHandler.compact(new CompactionRequest("default", tblName, CompactionType.MINOR)); Worker t = new Worker(); t.setThreadId((int) t.getId()); - t.setHiveConf(hiveConf); + t.setConf(hiveConf); AtomicBoolean stop = new AtomicBoolean(true); AtomicBoolean looped = new AtomicBoolean(); t.init(stop, looped); @@ -1241,14 +1241,14 @@ protected void writeBetweenWorkerAndCleanerForVariousTblProperties(String tblPro //run Cleaner Cleaner c = new Cleaner(); c.setThreadId((int)c.getId()); - c.setHiveConf(hiveConf); + c.setConf(hiveConf); c.init(stop, new AtomicBoolean()); c.run(); //this seems odd, but we wan to make sure that to run CompactionTxnHandler.cleanEmptyAbortedTxns() Initiator i = new Initiator(); i.setThreadId((int)i.getId()); - i.setHiveConf(hiveConf); + i.setConf(hiveConf); i.init(stop, new AtomicBoolean()); i.run(); @@ -1318,7 +1318,7 @@ public void testCompactWithDelete() throws Exception { runStatementOnDriver("alter table "+ Table.ACIDTBL + " compact 'MAJOR'"); Worker t = new Worker(); t.setThreadId((int) t.getId()); - t.setHiveConf(hiveConf); + t.setConf(hiveConf); AtomicBoolean stop = new AtomicBoolean(); AtomicBoolean looped = new AtomicBoolean(); stop.set(true); diff --git ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/CompactorTest.java ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/CompactorTest.java index 7455fa8449..af8a63d74e 100644 --- ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/CompactorTest.java +++ ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/CompactorTest.java @@ -280,7 +280,7 @@ private void startThread(char type, boolean stopAfterOne, AtomicBoolean looped) default: throw new RuntimeException("Huh? Unknown thread type."); } t.setThreadId((int) t.getId()); - t.setHiveConf(conf); + t.setConf(conf); stop.set(stopAfterOne); t.init(stop, looped); if (stopAfterOne) t.run(); diff --git standalone-metastore/pom.xml standalone-metastore/pom.xml index e38ef3770b..6879204170 100644 --- standalone-metastore/pom.xml +++ standalone-metastore/pom.xml @@ -43,6 +43,11 @@ ${guava.version} + com.google.protobuf + protobuf-java + ${protobuf.version} + + io.dropwizard.metrics metrics-core ${dropwizard.version} @@ -176,6 +181,39 @@ + + protobuf + + + + org.apache.maven.plugins + maven-antrun-plugin + + + generate-protobuf-sources + generate-sources + + + + + Building Protobuf + + + + + + + + + + run + + + + + + + @@ -202,6 +240,7 @@ src/gen/thrift/gen-javabean + src/gen/protobuf/gen-java @@ -225,6 +264,29 @@ + + org.apache.maven.plugins + maven-antrun-plugin + + + generate-version-annotation + generate-sources + + + + + + + + + + + + run + + + + diff --git metastore/src/gen/protobuf/gen-java/org/apache/hadoop/hive/metastore/Metastore.java standalone-metastore/src/gen/protobuf/gen-java/org/apache/hadoop/hive/metastore/Metastore.java similarity index 100% rename from metastore/src/gen/protobuf/gen-java/org/apache/hadoop/hive/metastore/Metastore.java rename to standalone-metastore/src/gen/protobuf/gen-java/org/apache/hadoop/hive/metastore/Metastore.java diff --git common/src/java/org/apache/hadoop/hive/common/classification/RetrySemantics.java standalone-metastore/src/main/java/org/apache/hadoop/hive/common/classification/RetrySemantics.java similarity index 94% rename from common/src/java/org/apache/hadoop/hive/common/classification/RetrySemantics.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/common/classification/RetrySemantics.java index f1c394620e..10d21eec33 100644 --- common/src/java/org/apache/hadoop/hive/common/classification/RetrySemantics.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/common/classification/RetrySemantics.java @@ -18,6 +18,9 @@ package org.apache.hadoop.hive.common.classification; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + import java.lang.annotation.ElementType; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; diff --git metastore/src/java/org/apache/hadoop/hive/metastore/AggregateStatsCache.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/AggregateStatsCache.java similarity index 91% rename from metastore/src/java/org/apache/hadoop/hive/metastore/AggregateStatsCache.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/AggregateStatsCache.java index 4ab178c606..bdac1618d6 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/AggregateStatsCache.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/AggregateStatsCache.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -19,10 +19,11 @@ package org.apache.hadoop.hive.metastore; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hive.common.util.BloomFilter; @@ -51,9 +52,9 @@ // Current nodes in the cache private final AtomicInteger currentNodes = new AtomicInteger(0); // Run the cleaner thread when the cache is maxFull% full - private final float maxFull; + private final double maxFull; // Run the cleaner thread until cache is cleanUntil% occupied - private final float cleanUntil; + private final double cleanUntil; // Nodes go stale after this private final long timeToLiveMs; // Max time when waiting for write locks on node list @@ -63,9 +64,9 @@ // Maximum number of paritions aggregated per cache node private final int maxPartsPerCacheNode; // Bloom filter false positive probability - private final float falsePositiveProbability; + private final double falsePositiveProbability; // Max tolerable variance for matches - private final float maxVariance; + private final double maxVariance; // Used to determine if cleaner thread is already running private boolean isCleaning = false; private final AtomicLong cacheHits = new AtomicLong(0); @@ -74,8 +75,8 @@ int numRemovedTTL = 0, numRemovedLRU = 0; private AggregateStatsCache(int maxCacheNodes, int maxPartsPerCacheNode, long timeToLiveMs, - float falsePositiveProbability, float maxVariance, long maxWriterWaitTime, - long maxReaderWaitTime, float maxFull, float cleanUntil) { + double falsePositiveProbability, double maxVariance, long maxWriterWaitTime, + long maxReaderWaitTime, double maxFull, double cleanUntil) { this.maxCacheNodes = maxCacheNodes; this.maxPartsPerCacheNode = maxPartsPerCacheNode; this.timeToLiveMs = timeToLiveMs; @@ -85,40 +86,33 @@ private AggregateStatsCache(int maxCacheNodes, int maxPartsPerCacheNode, long ti this.maxReaderWaitTime = maxReaderWaitTime; this.maxFull = maxFull; this.cleanUntil = cleanUntil; - this.cacheStore = new ConcurrentHashMap(); + this.cacheStore = new ConcurrentHashMap<>(); } public static synchronized AggregateStatsCache getInstance(Configuration conf) { if (self == null) { int maxCacheNodes = - HiveConf.getIntVar(conf, HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_SIZE); + MetastoreConf.getIntVar(conf, ConfVars.AGGREGATE_STATS_CACHE_SIZE); // The number of partitions aggregated per cache node // If the number of partitions requested is > this value, we'll fetch directly from Metastore int maxPartitionsPerCacheNode = - HiveConf - .getIntVar(conf, HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_MAX_PARTITIONS); - long timeToLiveMs = - HiveConf.getTimeVar(conf, HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_TTL, + MetastoreConf.getIntVar(conf, ConfVars.AGGREGATE_STATS_CACHE_MAX_PARTITIONS); + long timeToLiveMs = MetastoreConf.getTimeVar(conf, ConfVars.AGGREGATE_STATS_CACHE_TTL, TimeUnit.SECONDS)*1000; // False positives probability we are ready to tolerate for the underlying bloom filter - float falsePositiveProbability = - HiveConf.getFloatVar(conf, HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_FPP); + double falsePositiveProbability = + MetastoreConf.getDoubleVar(conf, ConfVars.AGGREGATE_STATS_CACHE_FPP); // Maximum tolerable variance in number of partitions between cached node and our request - float maxVariance = - HiveConf - .getFloatVar(conf, HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_MAX_VARIANCE); - long maxWriterWaitTime = - HiveConf.getTimeVar(conf, - HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_MAX_WRITER_WAIT, - TimeUnit.MILLISECONDS); - long maxReaderWaitTime = - HiveConf.getTimeVar(conf, - HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_MAX_READER_WAIT, - TimeUnit.MILLISECONDS); - float maxFull = - HiveConf.getFloatVar(conf, HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_MAX_FULL); - float cleanUntil = - HiveConf.getFloatVar(conf, HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_CLEAN_UNTIL); + double maxVariance = + MetastoreConf.getDoubleVar(conf, ConfVars.AGGREGATE_STATS_CACHE_MAX_VARIANCE); + long maxWriterWaitTime = MetastoreConf.getTimeVar(conf, + ConfVars.AGGREGATE_STATS_CACHE_MAX_WRITER_WAIT, TimeUnit.MILLISECONDS); + long maxReaderWaitTime = MetastoreConf.getTimeVar(conf, + ConfVars.AGGREGATE_STATS_CACHE_MAX_READER_WAIT, TimeUnit.MILLISECONDS); + double maxFull = + MetastoreConf.getDoubleVar(conf, ConfVars.AGGREGATE_STATS_CACHE_MAX_FULL); + double cleanUntil = + MetastoreConf.getDoubleVar(conf, ConfVars.AGGREGATE_STATS_CACHE_CLEAN_UNTIL); self = new AggregateStatsCache(maxCacheNodes, maxPartitionsPerCacheNode, timeToLiveMs, falsePositiveProbability, maxVariance, maxWriterWaitTime, maxReaderWaitTime, maxFull, @@ -143,7 +137,7 @@ public int getMaxPartsPerCacheNode() { return maxPartsPerCacheNode; } - public float getFalsePositiveProbability() { + public double getFalsePositiveProbability() { return falsePositiveProbability; } @@ -216,7 +210,7 @@ private AggrColStats findBestMatch(List partNames, List ca // Hits, misses tracked for a candidate node MatchStats matchStats; // MatchStats for each candidate - Map candidateMatchStats = new HashMap(); + Map candidateMatchStats = new HashMap<>(); // The final match we intend to return AggrColStats bestMatch = null; // To compare among potentially multiple matches @@ -294,7 +288,7 @@ public void add(String dbName, String tblName, String colName, long numPartsCach AggrColStats node = new AggrColStats(numPartsCached, bloomFilter, colStats); AggrColStatsList nodeList; AggrColStatsList newNodeList = new AggrColStatsList(); - newNodeList.nodes = new ArrayList(); + newNodeList.nodes = new ArrayList<>(); nodeList = cacheStore.putIfAbsent(key, newNodeList); if (nodeList == null) { nodeList = newNodeList; @@ -507,7 +501,7 @@ public String toString() { static class AggrColStatsList { // TODO: figure out a better data structure for node list(?) - private List nodes = new ArrayList(); + private List nodes = new ArrayList<>(); private final ReadWriteLock lock = new ReentrantReadWriteLock(); // Read lock for get operation private final Lock readLock = lock.readLock(); diff --git metastore/src/java/org/apache/hadoop/hive/metastore/DatabaseProduct.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/DatabaseProduct.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/DatabaseProduct.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/DatabaseProduct.java index 7634852a2f..03a5e4a349 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/DatabaseProduct.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/DatabaseProduct.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -72,4 +72,4 @@ public static boolean needsInBatching(DatabaseProduct dbType) { public static boolean hasJoinOperationOrderBug(DatabaseProduct dbType) { return dbType == DERBY || dbType == ORACLE; } -}; +} diff --git metastore/src/java/org/apache/hadoop/hive/metastore/DefaultHiveMetaHook.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/DefaultHiveMetaHook.java similarity index 100% rename from metastore/src/java/org/apache/hadoop/hive/metastore/DefaultHiveMetaHook.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/DefaultHiveMetaHook.java diff --git metastore/src/java/org/apache/hadoop/hive/metastore/DefaultMetaStoreFilterHookImpl.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/DefaultMetaStoreFilterHookImpl.java similarity index 96% rename from metastore/src/java/org/apache/hadoop/hive/metastore/DefaultMetaStoreFilterHookImpl.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/DefaultMetaStoreFilterHookImpl.java index f0cab26a0f..020bea8d13 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/DefaultMetaStoreFilterHookImpl.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/DefaultMetaStoreFilterHookImpl.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -20,7 +20,7 @@ import java.util.List; -import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.Index; import org.apache.hadoop.hive.metastore.api.MetaException; @@ -34,7 +34,7 @@ */ public class DefaultMetaStoreFilterHookImpl implements MetaStoreFilterHook { - public DefaultMetaStoreFilterHookImpl(HiveConf conf) { + public DefaultMetaStoreFilterHookImpl(Configuration conf) { } @Override diff --git metastore/src/java/org/apache/hadoop/hive/metastore/FileFormatProxy.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/FileFormatProxy.java similarity index 100% rename from metastore/src/java/org/apache/hadoop/hive/metastore/FileFormatProxy.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/FileFormatProxy.java diff --git metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaException.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaException.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaException.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaException.java index 05cc1fad96..5bd5a702b3 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaException.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaException.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaHook.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaHook.java similarity index 93% rename from metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaHook.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaHook.java index 3317bc51d4..2534fa2212 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaHook.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaHook.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,10 +18,9 @@ package org.apache.hadoop.hive.metastore; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.hive.metastore.api.MetaException; -import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.Table; /** diff --git metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaHookLoader.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaHookLoader.java similarity index 100% rename from metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaHookLoader.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaHookLoader.java diff --git metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java similarity index 93% rename from metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java index b7d7b50bb9..8322cb8d90 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -20,12 +20,13 @@ import java.io.FileNotFoundException; +import org.apache.hadoop.hive.metastore.utils.FileUtils; +import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.metastore.api.MetaException; public class HiveMetaStoreFsImpl implements MetaStoreFS { diff --git metastore/src/java/org/apache/hadoop/hive/metastore/IExtrapolatePartStatus.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IExtrapolatePartStatus.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/IExtrapolatePartStatus.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IExtrapolatePartStatus.java index d0569fb8d8..aa50e91076 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/IExtrapolatePartStatus.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IExtrapolatePartStatus.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git metastore/src/java/org/apache/hadoop/hive/metastore/IHMSHandler.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IHMSHandler.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/IHMSHandler.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IHMSHandler.java index cc7ae1d6d6..633b3c7a17 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/IHMSHandler.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IHMSHandler.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreSchemaInfo.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreSchemaInfo.java similarity index 98% rename from metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreSchemaInfo.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreSchemaInfo.java index 0a5f9789c3..be89f9b56b 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreSchemaInfo.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreSchemaInfo.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -17,10 +17,11 @@ */ package org.apache.hadoop.hive.metastore; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; import java.sql.Connection; import java.util.List; + +import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper; /** diff --git metastore/src/java/org/apache/hadoop/hive/metastore/LinearExtrapolatePartStatus.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/LinearExtrapolatePartStatus.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/LinearExtrapolatePartStatus.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/LinearExtrapolatePartStatus.java index f4e5ef7045..526b55b62d 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/LinearExtrapolatePartStatus.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/LinearExtrapolatePartStatus.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git metastore/src/java/org/apache/hadoop/hive/metastore/LockComponentBuilder.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/LockComponentBuilder.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/LockComponentBuilder.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/LockComponentBuilder.java index e07415274e..de6c718ba9 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/LockComponentBuilder.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/LockComponentBuilder.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git metastore/src/java/org/apache/hadoop/hive/metastore/LockRequestBuilder.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/LockRequestBuilder.java similarity index 98% rename from metastore/src/java/org/apache/hadoop/hive/metastore/LockRequestBuilder.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/LockRequestBuilder.java index 00a1907531..d03c73a4aa 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/LockRequestBuilder.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/LockRequestBuilder.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -108,7 +108,7 @@ public LockRequestBuilder addLockComponent(LockComponent component) { Map trie; LockTrie() { - trie = new LinkedHashMap(); + trie = new LinkedHashMap<>(); } public void add(LockComponent comp) { diff --git metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEndFunctionContext.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreEndFunctionContext.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEndFunctionContext.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreEndFunctionContext.java index ae75200e83..998531ffe6 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEndFunctionContext.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreEndFunctionContext.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEndFunctionListener.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreEndFunctionListener.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEndFunctionListener.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreEndFunctionListener.java index 429efab8db..bc7e0beb51 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEndFunctionListener.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreEndFunctionListener.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreFS.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreFS.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreFS.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreFS.java index 4f525a4cd6..ddcda4c41f 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreFS.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreFS.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreFilterHook.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreFilterHook.java similarity index 94% rename from metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreFilterHook.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreFilterHook.java index 2263837749..fb341b2ea1 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreFilterHook.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreFilterHook.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -20,8 +20,8 @@ import java.util.List; -import org.apache.hadoop.hive.common.classification.InterfaceAudience.LimitedPrivate; -import org.apache.hadoop.hive.common.classification.InterfaceStability.Evolving; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.Index; import org.apache.hadoop.hive.metastore.api.MetaException; @@ -35,8 +35,8 @@ * plugins on hiveserver2 to filter metadata results, especially in case of * non-impersonation mode where the metastore doesn't know the end user's identity. */ -@LimitedPrivate(value = { "Apache Sentry (Incubating)" }) -@Evolving +@InterfaceAudience.LimitedPrivate(value = {"Apache Sentry (Incubating)" }) +@InterfaceStability.Evolving public interface MetaStoreFilterHook { /** diff --git metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreInit.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreInit.java similarity index 84% rename from metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreInit.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreInit.java index 6123a1e5a3..d3eee8548e 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreInit.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreInit.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,11 +18,12 @@ package org.apache.hadoop.hive.metastore; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; +import org.apache.hadoop.hive.metastore.utils.JavaUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.common.JavaUtils; -import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.hooks.JDOConnectionURLHook; import org.apache.hadoop.util.ReflectionUtils; @@ -51,7 +52,7 @@ * configuration * @throws MetaException */ - static boolean updateConnectionURL(HiveConf originalConf, Configuration activeConf, + static boolean updateConnectionURL(Configuration originalConf, Configuration activeConf, String badUrl, MetaStoreInitData updateData) throws MetaException { String connectUrl = null; @@ -74,26 +75,23 @@ static boolean updateConnectionURL(HiveConf originalConf, Configuration activeCo if (connectUrl != null && !connectUrl.equals(currentUrl)) { LOG.error( String.format("Overriding %s with %s", - HiveConf.ConfVars.METASTORECONNECTURLKEY.toString(), + MetastoreConf.ConfVars.CONNECTURLKEY.toString(), connectUrl)); - activeConf.set(HiveConf.ConfVars.METASTORECONNECTURLKEY.toString(), - connectUrl); + MetastoreConf.setVar(activeConf, ConfVars.CONNECTURLKEY, connectUrl); return true; } return false; } static String getConnectionURL(Configuration conf) { - return conf.get( - HiveConf.ConfVars.METASTORECONNECTURLKEY.toString(), ""); + return MetastoreConf.getVar(conf, ConfVars.CONNECTURLKEY, ""); } // Multiple threads could try to initialize at the same time. - synchronized private static void initConnectionUrlHook(HiveConf hiveConf, + synchronized private static void initConnectionUrlHook(Configuration conf, MetaStoreInitData updateData) throws ClassNotFoundException { - String className = - hiveConf.get(HiveConf.ConfVars.METASTORECONNECTURLHOOK.toString(), "").trim(); + String className = MetastoreConf.getVar(conf, ConfVars.CONNECTURLHOOK, "").trim(); if (className.equals("")) { updateData.urlHookClassName = ""; updateData.urlHook = null; @@ -107,6 +105,5 @@ synchronized private static void initConnectionUrlHook(HiveConf hiveConf, JavaUtils.getClassLoader()); updateData.urlHook = (JDOConnectionURLHook) ReflectionUtils.newInstance(urlHookClass, null); } - return; } } diff --git metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java similarity index 97% rename from metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java index 8117ca1053..0c368555c6 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -32,9 +32,9 @@ import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper; import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper.MetaStoreConnectionInfo; -import org.apache.hive.common.util.HiveVersionInfo; import com.google.common.collect.ImmutableMap; +import org.apache.hadoop.hive.metastore.utils.MetastoreVersionInfo; public class MetaStoreSchemaInfo implements IMetaStoreSchemaInfo { @@ -64,7 +64,7 @@ public MetaStoreSchemaInfo(String hiveHome, String dbType) throws HiveMetaExcept private void loadAllUpgradeScripts(String dbType) throws HiveMetaException { // load upgrade order for the given dbType - List upgradeOrderList = new ArrayList(); + List upgradeOrderList = new ArrayList<>(); String upgradeListFile = getMetaStoreScriptDir() + File.separator + VERSION_UPGRADE_LIST + "." + dbType; try (FileReader fr = new FileReader(upgradeListFile); @@ -90,7 +90,7 @@ private void loadAllUpgradeScripts(String dbType) throws HiveMetaException { @Override public List getUpgradeScripts(String fromVersion) throws HiveMetaException { - List upgradeScriptList = new ArrayList(); + List upgradeScriptList = new ArrayList<>(); // check if we are already at current schema level if (getHiveSchemaVersion().equals(fromVersion)) { @@ -160,7 +160,7 @@ public String getPreUpgradeScriptName(int index, String upgradeScriptName) { @Override public String getHiveSchemaVersion() { - String hiveVersion = HiveVersionInfo.getShortVersion(); + String hiveVersion = MetastoreVersionInfo.getShortVersion(); return getEquivalentVersion(hiveVersion); } diff --git metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfoFactory.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfoFactory.java similarity index 89% rename from metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfoFactory.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfoFactory.java index 1133cf2449..d03b7ae7b7 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfoFactory.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfoFactory.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -21,7 +21,7 @@ import java.lang.reflect.InvocationTargetException; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,16 +42,15 @@ public static IMetaStoreSchemaInfo get(Configuration conf) { } public static IMetaStoreSchemaInfo get(Configuration conf, String hiveHome, String dbType) { - String className = conf.get(HiveConf.ConfVars.METASTORE_SCHEMA_INFO_CLASS.varname, - HiveConf.ConfVars.METASTORE_SCHEMA_INFO_CLASS.defaultStrVal); - Class clasz = null; + String className = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.SCHEMA_INFO_CLASS); + Class clasz; try { clasz = conf.getClassByName(className); } catch (ClassNotFoundException e) { LOG.error("Unable to load class " + className, e); throw new IllegalArgumentException(e); } - Constructor constructor = null; + Constructor constructor; try { constructor = clasz.getConstructor(String.class, String.class); constructor.setAccessible(true); diff --git metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreThread.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreThread.java similarity index 87% rename from metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreThread.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreThread.java index b62c45fa83..10f7732f88 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreThread.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreThread.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -17,7 +17,8 @@ */ package org.apache.hadoop.hive.metastore; -import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.api.MetaException; import java.util.concurrent.atomic.AtomicBoolean; @@ -25,23 +26,17 @@ /** * A thread that runs in the metastore, separate from the threads in the thrift service. */ -public interface MetaStoreThread { - - /** - * Set the Hive configuration for this thread. - * @param conf - */ - void setHiveConf(HiveConf conf); +public interface MetaStoreThread extends Configurable { /** * Set the id for this thread. - * @param threadId + * @param threadId id of the thread */ void setThreadId(int threadId); /** * Initialize the thread. This must not be called until after - * {@link #setHiveConf(org.apache.hadoop.hive.conf.HiveConf)} and {@link #setThreadId(int)} + * {@link #setConf(Configuration)} and {@link #setThreadId(int)} * have been called. * @param stop a flag to watch for when to stop. If this value is set to true, * the thread will terminate the next time through its main loop. diff --git metastore/src/java/org/apache/hadoop/hive/metastore/PartitionDropOptions.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/PartitionDropOptions.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/PartitionDropOptions.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/PartitionDropOptions.java index e8ffbd57f3..40018c91ad 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/PartitionDropOptions.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/PartitionDropOptions.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git metastore/src/java/org/apache/hadoop/hive/metastore/PartitionExpressionProxy.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/PartitionExpressionProxy.java similarity index 100% rename from metastore/src/java/org/apache/hadoop/hive/metastore/PartitionExpressionProxy.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/PartitionExpressionProxy.java diff --git metastore/src/java/org/apache/hadoop/hive/metastore/TServerSocketKeepAlive.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TServerSocketKeepAlive.java similarity index 100% rename from metastore/src/java/org/apache/hadoop/hive/metastore/TServerSocketKeepAlive.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TServerSocketKeepAlive.java diff --git metastore/src/java/org/apache/hadoop/hive/metastore/TableType.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TableType.java similarity index 100% rename from metastore/src/java/org/apache/hadoop/hive/metastore/TableType.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TableType.java diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/annotation/MetastoreVersionAnnotation.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/annotation/MetastoreVersionAnnotation.java new file mode 100644 index 0000000000..0542546277 --- /dev/null +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/annotation/MetastoreVersionAnnotation.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.annotation; + +import org.apache.hadoop.classification.InterfaceStability; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + + +/** + * HiveVersionAnnotation. + * + */ +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.PACKAGE) +@InterfaceStability.Unstable +public @interface MetastoreVersionAnnotation { + + /** + * Get the Hive version + * @return the version string "0.6.3-dev" + */ + String version(); + + /** + * Get the Hive short version containing major/minor/change version numbers + * @return the short version string "0.6.3" + */ + String shortVersion(); + + /** + * Get the username that compiled Hive. + */ + String user(); + + /** + * Get the date when Hive was compiled. + * @return the date in unix 'date' format + */ + String date(); + + /** + * Get the url for the git repository. + */ + String url(); + + /** + * Get the git revision. + * @return the revision number as a string (eg. "451451") + */ + String revision(); + + /** + * Get the branch from which this was compiled. + * @return The branch name, e.g. "trunk" or "branches/branch-0.20" + */ + String branch(); + + /** + * Get a checksum of the source files from which + * Hive was compiled. + * @return a string that uniquely identifies the source + **/ + String srcChecksum(); + +} diff --git metastore/src/java/org/apache/hadoop/hive/metastore/annotation/NoReconnect.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/annotation/NoReconnect.java similarity index 100% rename from metastore/src/java/org/apache/hadoop/hive/metastore/annotation/NoReconnect.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/annotation/NoReconnect.java diff --git metastore/src/java/org/apache/hadoop/hive/metastore/hooks/JDOConnectionURLHook.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/hooks/JDOConnectionURLHook.java similarity index 92% rename from metastore/src/java/org/apache/hadoop/hive/metastore/hooks/JDOConnectionURLHook.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/hooks/JDOConnectionURLHook.java index a1a2fb918a..65084bd7b6 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/hooks/JDOConnectionURLHook.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/hooks/JDOConnectionURLHook.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,9 +18,9 @@ package org.apache.hadoop.hive.metastore.hooks; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; /** * JDOConnectURLHook is used to get the URL that JDO uses to connect to the diff --git metastore/src/java/org/apache/hadoop/hive/metastore/partition/spec/CompositePartitionSpecProxy.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/CompositePartitionSpecProxy.java similarity index 96% rename from metastore/src/java/org/apache/hadoop/hive/metastore/partition/spec/CompositePartitionSpecProxy.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/CompositePartitionSpecProxy.java index 7e94e34295..e34335d3e1 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/partition/spec/CompositePartitionSpecProxy.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/CompositePartitionSpecProxy.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,8 +18,6 @@ package org.apache.hadoop.hive.metastore.partition.spec; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.PartitionSpec; @@ -48,7 +46,7 @@ protected CompositePartitionSpecProxy(List partitionSpecs) { else { dbName = partitionSpecs.get(0).getDbName(); tableName = partitionSpecs.get(0).getTableName(); - this.partitionSpecProxies = new ArrayList(partitionSpecs.size()); + this.partitionSpecProxies = new ArrayList<>(partitionSpecs.size()); for (PartitionSpec partitionSpec : partitionSpecs) { PartitionSpecProxy partitionSpecProxy = Factory.get(partitionSpec); this.partitionSpecProxies.add(partitionSpecProxy); @@ -63,7 +61,7 @@ protected CompositePartitionSpecProxy(String dbName, String tableName, List(partitionSpecs.size()); + this.partitionSpecProxies = new ArrayList<>(partitionSpecs.size()); for (PartitionSpec partitionSpec : partitionSpecs) { this.partitionSpecProxies.add(PartitionSpecProxy.Factory.get(partitionSpec)); } diff --git metastore/src/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionListComposingSpecProxy.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionListComposingSpecProxy.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionListComposingSpecProxy.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionListComposingSpecProxy.java index 154011ebf5..7b0550bfc1 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionListComposingSpecProxy.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionListComposingSpecProxy.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git metastore/src/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionSpecProxy.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionSpecProxy.java similarity index 93% rename from metastore/src/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionSpecProxy.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionSpecProxy.java index fdb086749b..2640a241ab 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionSpecProxy.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionSpecProxy.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -121,57 +121,57 @@ public static PartitionSpecProxy get(List partitionSpecs) { /** * Iterator to iterate over Partitions corresponding to a PartitionSpec. */ - public static interface PartitionIterator extends java.util.Iterator { + public interface PartitionIterator extends java.util.Iterator { /** * Getter for the Partition "pointed to" by the iterator. * Like next(), but without advancing the iterator. * @return The "current" partition object. */ - public Partition getCurrent(); + Partition getCurrent(); /** * Getter for the name of the DB. * @return Name of the DB. */ - public String getDbName(); + String getDbName(); /** * Getter for the name of the table. * @return Name of the table. */ - public String getTableName(); + String getTableName(); /** * Getter for the Partition parameters. * @return Key-value map for Partition-level parameters. */ - public Map getParameters(); + Map getParameters(); /** * Setter for Partition parameters. * @param parameters Key-value map fo Partition-level parameters. */ - public void setParameters(Map parameters); + void setParameters(Map parameters); /** * Insert an individual parameter to a Partition's parameter-set. * @param key * @param value */ - public void putToParameters(String key, String value); + void putToParameters(String key, String value); /** * Getter for Partition-location. * @return Partition's location. */ - public String getLocation(); + String getLocation(); /** * Setter for creation-time of a Partition. * @param time Timestamp indicating the time of creation of the Partition. */ - public void setCreateTime(long time); + void setCreateTime(long time); } // class PartitionIterator; diff --git metastore/src/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionSpecWithSharedSDProxy.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionSpecWithSharedSDProxy.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionSpecWithSharedSDProxy.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionSpecWithSharedSDProxy.java index 6a3e147eea..36b05f7153 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionSpecWithSharedSDProxy.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionSpecWithSharedSDProxy.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveSchemaHelper.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/HiveSchemaHelper.java similarity index 89% rename from metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveSchemaHelper.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/HiveSchemaHelper.java index 620ea5ffff..4ba11b8abd 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveSchemaHelper.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/HiveSchemaHelper.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -19,8 +19,9 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; -import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.HiveMetaException; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import java.io.BufferedReader; import java.io.File; @@ -46,19 +47,19 @@ * @param userName metastore connection username * @param password metastore connection password * @param printInfo print connection parameters - * @param hiveConf hive config object + * @param conf hive config object * @return metastore connection object * @throws org.apache.hadoop.hive.metastore.HiveMetaException */ public static Connection getConnectionToMetastore(String userName, String password, String url, String driver, boolean printInfo, - HiveConf hiveConf) + Configuration conf) throws HiveMetaException { try { url = url == null ? getValidConfVar( - HiveConf.ConfVars.METASTORECONNECTURLKEY, hiveConf) : url; + MetastoreConf.ConfVars.CONNECTURLKEY, conf) : url; driver = driver == null ? getValidConfVar( - HiveConf.ConfVars.METASTORE_CONNECTION_DRIVER, hiveConf) : driver; + MetastoreConf.ConfVars.CONNECTION_DRIVER, conf) : driver; if (printInfo) { System.out.println("Metastore connection URL:\t " + url); System.out.println("Metastore Connection Driver :\t " + driver); @@ -84,12 +85,12 @@ public static Connection getConnectionToMetastore(String userName, public static Connection getConnectionToMetastore(MetaStoreConnectionInfo info) throws HiveMetaException { return getConnectionToMetastore(info.getUsername(), info.getPassword(), info.getUrl(), - info.getDriver(), info.getPrintInfo(), info.getHiveConf()); + info.getDriver(), info.getPrintInfo(), info.getConf()); } - public static String getValidConfVar(HiveConf.ConfVars confVar, HiveConf hiveConf) + public static String getValidConfVar(MetastoreConf.ConfVars confVar, Configuration conf) throws IOException { - String confVarStr = hiveConf.get(confVar.varname); + String confVarStr = conf.get(confVar.varname); if (confVarStr == null || confVarStr.isEmpty()) { throw new IOException("Empty " + confVar.varname); } @@ -98,14 +99,14 @@ public static String getValidConfVar(HiveConf.ConfVars confVar, HiveConf hiveCon public interface NestedScriptParser { - public enum CommandType { + enum CommandType { PARTIAL_STATEMENT, TERMINATED_STATEMENT, COMMENT } - static final String DEFAULT_DELIMITER = ";"; - static final String DEFAULT_QUOTE = "\""; + String DEFAULT_DELIMITER = ";"; + String DEFAULT_QUOTE = "\""; /** * Find the type of given command @@ -113,7 +114,7 @@ public static String getValidConfVar(HiveConf.ConfVars confVar, HiveConf hiveCon * @param dbCommand * @return */ - public boolean isPartialCommand(String dbCommand) throws IllegalArgumentException; + boolean isPartialCommand(String dbCommand) throws IllegalArgumentException; /** * Parse the DB specific nesting format and extract the inner script name if any @@ -122,7 +123,7 @@ public static String getValidConfVar(HiveConf.ConfVars confVar, HiveConf hiveCon * @return * @throws IllegalFormatException */ - public String getScriptName(String dbCommand) throws IllegalArgumentException; + String getScriptName(String dbCommand) throws IllegalArgumentException; /** * Find if the given command is a nested script execution @@ -130,7 +131,7 @@ public static String getValidConfVar(HiveConf.ConfVars confVar, HiveConf hiveCon * @param dbCommand * @return */ - public boolean isNestedScript(String dbCommand); + boolean isNestedScript(String dbCommand); /** * Find if the given command should not be passed to DB @@ -138,35 +139,35 @@ public static String getValidConfVar(HiveConf.ConfVars confVar, HiveConf hiveCon * @param dbCommand * @return */ - public boolean isNonExecCommand(String dbCommand); + boolean isNonExecCommand(String dbCommand); /** * Get the SQL statement delimiter * * @return */ - public String getDelimiter(); + String getDelimiter(); /** * Get the SQL indentifier quotation character * * @return */ - public String getQuoteCharacter(); + String getQuoteCharacter(); /** * Clear any client specific tags * * @return */ - public String cleanseCommand(String dbCommand); + String cleanseCommand(String dbCommand); /** * Does the DB required table/column names quoted * * @return */ - public boolean needsQuotedIdentifier(); + boolean needsQuotedIdentifier(); /** * Flatten the nested upgrade script into a buffer @@ -175,7 +176,7 @@ public static String getValidConfVar(HiveConf.ConfVars confVar, HiveConf hiveCon * @param scriptFile upgrade script file * @return string of sql commands */ - public String buildCommand(String scriptDir, String scriptFile) + String buildCommand(String scriptDir, String scriptFile) throws IllegalFormatException, IOException; /** @@ -186,7 +187,7 @@ public String buildCommand(String scriptDir, String scriptFile) * @param fixQuotes whether to replace quote characters * @return string of sql commands */ - public String buildCommand(String scriptDir, String scriptFile, boolean fixQuotes) + String buildCommand(String scriptDir, String scriptFile, boolean fixQuotes) throws IllegalFormatException, IOException; } @@ -199,14 +200,14 @@ public String buildCommand(String scriptDir, String scriptFile, boolean fixQuote private List dbOpts; private String msUsername; private String msPassword; - private HiveConf hiveConf; + private Configuration conf; public AbstractCommandParser(String dbOpts, String msUsername, String msPassword, - HiveConf hiveConf) { + Configuration conf) { setDbOpts(dbOpts); this.msUsername = msUsername; this.msPassword = msPassword; - this.hiveConf = hiveConf; + this.conf = conf; } @Override @@ -328,8 +329,8 @@ protected String getMsPassword() { return msPassword; } - protected HiveConf getHiveConf() { - return hiveConf; + protected Configuration getConf() { + return conf; } } @@ -338,8 +339,8 @@ protected HiveConf getHiveConf() { private static final String DERBY_NESTING_TOKEN = "RUN"; public DerbyCommandParser(String dbOpts, String msUsername, String msPassword, - HiveConf hiveConf) { - super(dbOpts, msUsername, msPassword, hiveConf); + Configuration conf) { + super(dbOpts, msUsername, msPassword, conf); } @Override @@ -368,8 +369,8 @@ public boolean isNestedScript(String dbCommand) { private final NestedScriptParser nestedDbCommandParser; public HiveCommandParser(String dbOpts, String msUsername, String msPassword, - HiveConf hiveConf, String metaDbType) { - super(dbOpts, msUsername, msPassword, hiveConf); + Configuration conf, String metaDbType) { + super(dbOpts, msUsername, msPassword, conf); nestedDbCommandParser = getDbCommandParser(metaDbType); } @@ -404,8 +405,8 @@ public boolean isNestedScript(String dbCommand) { private String delimiter = DEFAULT_DELIMITER; public MySqlCommandParser(String dbOpts, String msUsername, String msPassword, - HiveConf hiveConf) { - super(dbOpts, msUsername, msPassword, hiveConf); + Configuration conf) { + super(dbOpts, msUsername, msPassword, conf); } @Override @@ -470,8 +471,8 @@ public String cleanseCommand(String dbCommand) { public static final String POSTGRES_SKIP_STANDARD_STRINGS_DBOPT = "postgres.filter.81"; public PostgresCommandParser(String dbOpts, String msUsername, String msPassword, - HiveConf hiveConf) { - super(dbOpts, msUsername, msPassword, hiveConf); + Configuration conf) { + super(dbOpts, msUsername, msPassword, conf); } @Override @@ -513,8 +514,8 @@ public boolean isNonExecCommand(String dbCommand) { private static final String ORACLE_NESTING_TOKEN = "@"; public OracleCommandParser(String dbOpts, String msUsername, String msPassword, - HiveConf hiveConf) { - super(dbOpts, msUsername, msPassword, hiveConf); + Configuration conf) { + super(dbOpts, msUsername, msPassword, conf); } @Override @@ -537,8 +538,8 @@ public boolean isNestedScript(String dbCommand) { private static final String MSSQL_NESTING_TOKEN = ":r"; public MSSQLCommandParser(String dbOpts, String msUsername, String msPassword, - HiveConf hiveConf) { - super(dbOpts, msUsername, msPassword, hiveConf); + Configuration conf) { + super(dbOpts, msUsername, msPassword, conf); } @Override @@ -566,19 +567,19 @@ public static NestedScriptParser getDbCommandParser(String dbName, String metaDb public static NestedScriptParser getDbCommandParser(String dbName, String dbOpts, String msUsername, String msPassword, - HiveConf hiveConf, String metaDbType) { + Configuration conf, String metaDbType) { if (dbName.equalsIgnoreCase(DB_DERBY)) { - return new DerbyCommandParser(dbOpts, msUsername, msPassword, hiveConf); + return new DerbyCommandParser(dbOpts, msUsername, msPassword, conf); } else if (dbName.equalsIgnoreCase(DB_HIVE)) { - return new HiveCommandParser(dbOpts, msUsername, msPassword, hiveConf, metaDbType); + return new HiveCommandParser(dbOpts, msUsername, msPassword, conf, metaDbType); } else if (dbName.equalsIgnoreCase(DB_MSSQL)) { - return new MSSQLCommandParser(dbOpts, msUsername, msPassword, hiveConf); + return new MSSQLCommandParser(dbOpts, msUsername, msPassword, conf); } else if (dbName.equalsIgnoreCase(DB_MYSQL)) { - return new MySqlCommandParser(dbOpts, msUsername, msPassword, hiveConf); + return new MySqlCommandParser(dbOpts, msUsername, msPassword, conf); } else if (dbName.equalsIgnoreCase(DB_POSTGRACE)) { - return new PostgresCommandParser(dbOpts, msUsername, msPassword, hiveConf); + return new PostgresCommandParser(dbOpts, msUsername, msPassword, conf); } else if (dbName.equalsIgnoreCase(DB_ORACLE)) { - return new OracleCommandParser(dbOpts, msUsername, msPassword, hiveConf); + return new OracleCommandParser(dbOpts, msUsername, msPassword, conf); } else { throw new IllegalArgumentException("Unknown dbType " + dbName); } @@ -590,18 +591,18 @@ public static NestedScriptParser getDbCommandParser(String dbName, private final String url; private final String driver; private final boolean printInfo; - private final HiveConf hiveConf; + private final Configuration conf; private final String dbType; public MetaStoreConnectionInfo(String userName, String password, String url, String driver, - boolean printInfo, HiveConf hiveConf, String dbType) { + boolean printInfo, Configuration conf, String dbType) { super(); this.userName = userName; this.password = password; this.url = url; this.driver = driver; this.printInfo = printInfo; - this.hiveConf = hiveConf; + this.conf = conf; this.dbType = dbType; } @@ -621,8 +622,8 @@ public boolean isPrintInfo() { return printInfo; } - public HiveConf getHiveConf() { - return hiveConf; + public Configuration getConf() { + return conf; } public String getUsername() { diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/FileUtils.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/FileUtils.java new file mode 100644 index 0000000000..2310df6c82 --- /dev/null +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/FileUtils.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.utils; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.Trash; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; + +public class FileUtils { + private static final Logger LOG = LoggerFactory.getLogger(FileUtils.class); + + /** + * Move a particular file or directory to the trash. + * @param fs FileSystem to use + * @param f path of file or directory to move to trash. + * @param conf + * @return true if move successful + * @throws IOException + */ + public static boolean moveToTrash(FileSystem fs, Path f, Configuration conf, boolean purge) + throws IOException { + LOG.debug("deleting " + f); + boolean result = false; + try { + if(purge) { + LOG.debug("purge is set to true. Not moving to Trash " + f); + } else { + result = Trash.moveToAppropriateTrash(fs, f, conf); + if (result) { + LOG.trace("Moved to trash: " + f); + return true; + } + } + } catch (IOException ioe) { + // for whatever failure reason including that trash has lower encryption zone + // retry with force delete + LOG.warn(ioe.getMessage() + "; Force to delete it."); + } + + result = fs.delete(f, true); + if (!result) { + LOG.error("Failed to delete " + f); + } + return result; + } +} diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/JavaUtils.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/JavaUtils.java new file mode 100644 index 0000000000..81f8a8518d --- /dev/null +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/JavaUtils.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.utils; + +public class JavaUtils { + /** + * Standard way of getting classloader in Hive code (outside of Hadoop). + * + * Uses the context loader to get access to classpaths to auxiliary and jars + * added with 'add jar' command. Falls back to current classloader. + * + * In Hadoop-related code, we use Configuration.getClassLoader(). + * @return the class loader + */ + public static ClassLoader getClassLoader() { + ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); + if (classLoader == null) { + classLoader = JavaUtils.class.getClassLoader(); + } + return classLoader; + } +} diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java new file mode 100644 index 0000000000..3ef7e514fd --- /dev/null +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.utils; + +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MetaStoreUtils { + private static final Logger LOG = LoggerFactory.getLogger(MetaStoreUtils.class); + + /** + * Catches exceptions that can't be handled and bundles them to MetaException + * + * @param e exception to wrap. + * @throws MetaException wrapper for the exception + */ + public static void logAndThrowMetaException(Exception e) throws MetaException { + String exInfo = "Got exception: " + e.getClass().getName() + " " + + e.getMessage(); + LOG.error(exInfo, e); + LOG.error("Converting exception to MetaException"); + throw new MetaException(exInfo); + } + +} diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetastoreVersionInfo.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetastoreVersionInfo.java new file mode 100644 index 0000000000..45ba847ee9 --- /dev/null +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetastoreVersionInfo.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.utils; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.annotation.MetastoreVersionAnnotation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * HiveVersionInfo. + * + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +public class MetastoreVersionInfo { + private static final Logger LOG = LoggerFactory.getLogger(MetastoreVersionInfo.class); + + private static Package myPackage; + private static MetastoreVersionAnnotation version; + + static { + myPackage = MetastoreVersionAnnotation.class.getPackage(); + version = myPackage.getAnnotation(MetastoreVersionAnnotation.class); + } + + /** + * Get the meta-data for the Hive package. + * @return + */ + static Package getPackage() { + return myPackage; + } + + /** + * Get the Hive version. + * @return the Hive version string, eg. "0.6.3-dev" + */ + public static String getVersion() { + return version != null ? version.version() : "Unknown"; + } + + /** + * Get the Hive short version, with major/minor/change version numbers. + * @return short version string, eg. "0.6.3" + */ + public static String getShortVersion() { + return version != null ? version.shortVersion() : "Unknown"; + } + + /** + * Get the git revision number for the root directory + * @return the revision number, eg. "451451" + */ + public static String getRevision() { + return version != null ? version.revision() : "Unknown"; + } + + /** + * Get the branch on which this originated. + * @return The branch name, e.g. "trunk" or "branches/branch-0.20" + */ + public static String getBranch() { + return version != null ? version.branch() : "Unknown"; + } + + /** + * The date that Hive was compiled. + * @return the compilation date in unix date format + */ + public static String getDate() { + return version != null ? version.date() : "Unknown"; + } + + /** + * The user that compiled Hive. + * @return the username of the user + */ + public static String getUser() { + return version != null ? version.user() : "Unknown"; + } + + /** + * Get the git URL for the root Hive directory. + */ + public static String getUrl() { + return version != null ? version.url() : "Unknown"; + } + + /** + * Get the checksum of the source files from which Hive was + * built. + **/ + public static String getSrcChecksum() { + return version != null ? version.srcChecksum() : "Unknown"; + } + + /** + * Returns the buildVersion which includes version, + * revision, user and date. + */ + public static String getBuildVersion(){ + return MetastoreVersionInfo.getVersion() + + " from " + MetastoreVersionInfo.getRevision() + + " by " + MetastoreVersionInfo.getUser() + + " source checksum " + MetastoreVersionInfo.getSrcChecksum(); + } + + public static void main(String[] args) { + LOG.debug("version: "+ version); + System.out.println("Hive " + getVersion()); + System.out.println("Git " + getUrl() + " -r " + getRevision()); + System.out.println("Compiled by " + getUser() + " on " + getDate()); + System.out.println("From source with checksum " + getSrcChecksum()); + } + +} diff --git metastore/src/protobuf/org/apache/hadoop/hive/metastore/metastore.proto standalone-metastore/src/main/protobuf/org/apache/hadoop/hive/metastore/metastore.proto similarity index 100% rename from metastore/src/protobuf/org/apache/hadoop/hive/metastore/metastore.proto rename to standalone-metastore/src/main/protobuf/org/apache/hadoop/hive/metastore/metastore.proto diff --git standalone-metastore/src/main/resources/saveVersion.sh standalone-metastore/src/main/resources/saveVersion.sh new file mode 100644 index 0000000000..c9bde68835 --- /dev/null +++ standalone-metastore/src/main/resources/saveVersion.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# This file is used to generate the package-info.java class that +# records the version, revision, branch, user, timestamp, and url +unset LANG +unset LC_CTYPE +unset LC_TIME +version=$1 +shortversion=$2 +src_dir=$3 +revision=$4 +branch=$5 +url=$6 +user=`whoami` +date=`date` +dir=`pwd` +cwd=`dirname $dir` +if [ "$revision" = "" ]; then + if git rev-parse HEAD 2>/dev/null > /dev/null ; then + revision=`git log -1 --pretty=format:"%H"` + hostname=`hostname` + branch=`git branch | sed -n -e 's/^* //p'` + url="git://${hostname}${cwd}" + elif [ -d .svn ]; then + revision=`svn info ../ | sed -n -e 's/Last Changed Rev: \(.*\)/\1/p'` + url=`svn info ../ | sed -n -e 's/^URL: \(.*\)/\1/p'` + # Get canonical branch (branches/X, tags/X, or trunk) + branch=`echo $url | sed -n -e 's,.*\(branches/.*\)$,\1,p' \ + -e 's,.*\(tags/.*\)$,\1,p' \ + -e 's,.*trunk$,trunk,p'` + else + revision="Unknown" + branch="Unknown" + url="file://$cwd" + fi +fi +if [ "$branch" = "" ]; then + branch="Unknown" +fi +if [ "$url" = "" ]; then + url="file://$cwd" +fi + +if [ -x /sbin/md5 ]; then + md5="/sbin/md5" +else + md5="md5sum" +fi + +srcChecksum=`find ../ -name '*.java' | grep -v generated-sources | LC_ALL=C sort | xargs $md5 | $md5 | cut -d ' ' -f 1` + +mkdir -p $src_dir/gen/org/apache/hadoop/hive/metastore/annotation + +# In Windows, all the following string ends with \r, need to get rid of them +branch=`echo $branch | tr -d '\r'` +user=`echo $user | tr -d '\r'` +date=`echo $date | tr -d '\r'` +url=`echo $url | tr -d '\r'` +srcChecksum=`echo $srcChecksum | tr -d '\r'` + +cat << EOF | \ + sed -e "s/VERSION/$version/" -e "s/SHORTVERSION/$shortversion/" \ + -e "s/USER/$user/" -e "s/DATE/$date/" \ + -e "s|URL|$url|" -e "s/REV/$revision/" \ + -e "s|BRANCH|$branch|" -e "s/SRCCHECKSUM/$srcChecksum/" \ + > $src_dir/gen/org/apache/hadoop/hive/metastore/annotation/package-info.java +/* + * Generated by saveVersion.sh + */ +@MetastoreVersionAnnotation(version="VERSION", shortVersion="SHORTVERSION", + revision="REV", branch="BRANCH", + user="USER", date="DATE", url="URL", + srcChecksum="SRCCHECKSUM") +package org.apache.hadoop.hive.metastore.annotation; +EOF diff --git metastore/src/test/org/apache/hadoop/hive/metastore/TestAggregateStatsCache.java standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestAggregateStatsCache.java similarity index 85% rename from metastore/src/test/org/apache/hadoop/hive/metastore/TestAggregateStatsCache.java rename to standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestAggregateStatsCache.java index 40700daab8..6e06026f77 100644 --- metastore/src/test/org/apache/hadoop/hive/metastore/TestAggregateStatsCache.java +++ standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestAggregateStatsCache.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -21,13 +21,16 @@ import java.util.ArrayList; import java.util.List; +import java.util.concurrent.TimeUnit; -import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.metastore.api.LongColumnStatsData; import org.apache.hadoop.hive.metastore.AggregateStatsCache.AggrColStats; import org.apache.hadoop.hive.metastore.AggregateStatsCache.Key; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; import org.apache.hive.common.util.BloomFilter; import org.junit.After; import org.junit.AfterClass; @@ -46,15 +49,15 @@ static int NUM_COLS = 5; static int MAX_CACHE_NODES = 10; static int MAX_PARTITIONS_PER_CACHE_NODE = 10; - static String TIME_TO_LIVE = "20s"; - static String MAX_WRITER_WAIT = "1s"; - static String MAX_READER_WAIT = "1s"; - static float FALSE_POSITIVE_PROBABILITY = (float) 0.01; - static float MAX_VARIANCE = (float) 0.5; + static long TIME_TO_LIVE = 2; + static long MAX_WRITER_WAIT = 1; + static long MAX_READER_WAIT = 1; + static double FALSE_POSITIVE_PROBABILITY = 0.01; + static double MAX_VARIANCE = 0.5; static AggregateStatsCache cache; - static List tables = new ArrayList(); - static List tabParts = new ArrayList(); - static List tabCols = new ArrayList(); + static List tables = new ArrayList<>(); + static List tabParts = new ArrayList<>(); + static List tabCols = new ArrayList<>(); @BeforeClass public static void beforeTest() { @@ -91,20 +94,18 @@ public static void afterTest() { @Before public void setUp() { - HiveConf hiveConf = new HiveConf(); - hiveConf.setIntVar(HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_SIZE, - MAX_CACHE_NODES); - hiveConf.setIntVar(HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_MAX_PARTITIONS, + Configuration conf = MetastoreConf.newMetastoreConf(); + MetastoreConf.setLongVar(conf, ConfVars.AGGREGATE_STATS_CACHE_SIZE, MAX_CACHE_NODES); + MetastoreConf.setLongVar(conf, ConfVars.AGGREGATE_STATS_CACHE_MAX_PARTITIONS, MAX_PARTITIONS_PER_CACHE_NODE); - hiveConf.setFloatVar( - HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_FPP, - FALSE_POSITIVE_PROBABILITY); - hiveConf.setFloatVar(HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_MAX_VARIANCE, - MAX_VARIANCE); - hiveConf.setVar(HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_TTL, TIME_TO_LIVE); - hiveConf.setVar(HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_MAX_WRITER_WAIT, MAX_WRITER_WAIT); - hiveConf.setVar(HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_MAX_READER_WAIT, MAX_READER_WAIT); - cache = AggregateStatsCache.getInstance(hiveConf); + MetastoreConf.setDoubleVar(conf, ConfVars.AGGREGATE_STATS_CACHE_FPP, FALSE_POSITIVE_PROBABILITY); + MetastoreConf.setDoubleVar(conf, ConfVars.AGGREGATE_STATS_CACHE_MAX_VARIANCE, MAX_VARIANCE); + MetastoreConf.setTimeVar(conf, ConfVars.AGGREGATE_STATS_CACHE_TTL, TIME_TO_LIVE, TimeUnit.SECONDS); + MetastoreConf.setTimeVar(conf, ConfVars.AGGREGATE_STATS_CACHE_MAX_WRITER_WAIT, + MAX_WRITER_WAIT, TimeUnit.SECONDS); + MetastoreConf.setTimeVar(conf, ConfVars.AGGREGATE_STATS_CACHE_MAX_READER_WAIT, + MAX_READER_WAIT, TimeUnit.SECONDS); + cache = AggregateStatsCache.getInstance(conf); } @After @@ -204,10 +205,10 @@ public void testTimeToLive() throws Exception { // Now add to cache cache.add(DB_NAME, tblName, colName, 10, aggrColStats, bloomFilter); - // Sleep for 30 seconds - Thread.sleep(30000); + // Sleep for 3 seconds + Thread.sleep(3000); - // Get should fail now (since TTL is 20s) and we've snoozed for 30 seconds + // Get should fail now (since TTL is 2s) and we've snoozed for 3 seconds AggrColStats aggrStatsCached = cache.get(DB_NAME, tblName, colName, partNames); Assert.assertNull(aggrStatsCached); } @@ -227,7 +228,7 @@ public void testTimeToLive() throws Exception { if ((minPart < 1) || (maxPart > NUM_PARTS)) { throw new Exception("tabParts does not have these partition numbers"); } - List partNames = new ArrayList(); + List partNames = new ArrayList<>(); for (int i = minPart; i <= maxPart; i++) { String partName = tabParts.get(i-1); partNames.add(tabName + partName); diff --git metastore/src/test/org/apache/hadoop/hive/metastore/TestLockRequestBuilder.java standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestLockRequestBuilder.java similarity index 99% rename from metastore/src/test/org/apache/hadoop/hive/metastore/TestLockRequestBuilder.java rename to standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestLockRequestBuilder.java index 0964826934..a47c8a6d42 100644 --- metastore/src/test/org/apache/hadoop/hive/metastore/TestLockRequestBuilder.java +++ standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestLockRequestBuilder.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -17,11 +17,11 @@ */ package org.apache.hadoop.hive.metastore; -import junit.framework.Assert; import org.apache.hadoop.hive.metastore.api.LockComponent; import org.apache.hadoop.hive.metastore.api.LockLevel; import org.apache.hadoop.hive.metastore.api.LockRequest; import org.apache.hadoop.hive.metastore.api.LockType; +import org.junit.Assert; import org.junit.Test; import java.net.InetAddress; diff --git metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreSchemaFactory.java standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreSchemaFactory.java similarity index 81% rename from metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreSchemaFactory.java rename to standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreSchemaFactory.java index 99eec40406..1dac3ed4f2 100644 --- metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreSchemaFactory.java +++ standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreSchemaFactory.java @@ -17,17 +17,19 @@ */ package org.apache.hadoop.hive.metastore; -import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; import org.junit.Assert; import org.junit.Before; import org.junit.Test; public class TestMetaStoreSchemaFactory { - private HiveConf conf; + private Configuration conf; @Before public void setup() { - conf = new HiveConf(this.getClass()); + conf = MetastoreConf.newMetastoreConf(); } @Test @@ -38,7 +40,7 @@ public void testDefaultConfig() { @Test public void testWithConfigSet() { - conf.set(HiveConf.ConfVars.METASTORE_SCHEMA_INFO_CLASS.varname, + MetastoreConf.setVar(conf, ConfVars.SCHEMA_INFO_CLASS, MetaStoreSchemaInfo.class.getCanonicalName()); IMetaStoreSchemaInfo metastoreSchemaInfo = MetaStoreSchemaInfoFactory.get(conf); Assert.assertNotNull(metastoreSchemaInfo); @@ -48,7 +50,7 @@ public void testWithConfigSet() { @Test public void testConstructor() { - String className = conf.get(HiveConf.ConfVars.METASTORE_SCHEMA_INFO_CLASS.varname, + String className = MetastoreConf.getVar(conf, ConfVars.SCHEMA_INFO_CLASS, MetaStoreSchemaInfo.class.getCanonicalName()); Class clasz = null; try { @@ -61,7 +63,7 @@ public void testConstructor() { @Test(expected = IllegalArgumentException.class) public void testInvalidClassName() { - conf.set(HiveConf.ConfVars.METASTORE_SCHEMA_INFO_CLASS.varname, "invalid.class.name"); + MetastoreConf.setVar(conf, ConfVars.SCHEMA_INFO_CLASS, "invalid.class.name"); MetaStoreSchemaInfoFactory.get(conf); } } diff --git metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreSchemaInfo.java standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreSchemaInfo.java similarity index 93% rename from metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreSchemaInfo.java rename to standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreSchemaInfo.java index 35e862f10a..8485e997b5 100644 --- metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreSchemaInfo.java +++ standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreSchemaInfo.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,7 +18,7 @@ package org.apache.hadoop.hive.metastore; -import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.junit.Assert; import org.junit.Test; @@ -33,7 +33,7 @@ public void testIsVersionCompatible() throws Exception { // greater than or equal to it // check the compatible case IMetaStoreSchemaInfo metastoreSchemaInfo = - MetaStoreSchemaInfoFactory.get(new HiveConf(TestMetaStoreSchemaInfo.class)); + MetaStoreSchemaInfoFactory.get(MetastoreConf.newMetastoreConf()); Assert.assertTrue(metastoreSchemaInfo.isVersionCompatible("0.0.1", "0.0.1")); Assert.assertTrue(metastoreSchemaInfo.isVersionCompatible("0.0.1", "0.0.2")); Assert.assertTrue(metastoreSchemaInfo.isVersionCompatible("1.0.2", "2.0.1"));