diff --git hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/NotificationListener.java hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/NotificationListener.java index f7c2f446cf..e7289a7c1e 100644 --- hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/NotificationListener.java +++ hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/NotificationListener.java @@ -36,6 +36,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.MetaStoreEventListener; import org.apache.hadoop.hive.metastore.api.InvalidOperationException; import org.apache.hadoop.hive.metastore.api.MetaException; @@ -54,6 +55,7 @@ import org.apache.hive.hcatalog.common.HCatConstants; import org.apache.hive.hcatalog.messaging.HCatEventMessage; import org.apache.hive.hcatalog.messaging.MessageFactory; +import org.apache.thrift.TException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -182,7 +184,7 @@ public void onCreateDatabase(CreateDatabaseEvent dbEvent) // by listening on a topic named "HCAT" and message selector string // as "HCAT_EVENT = HCAT_ADD_DATABASE" if (dbEvent.getStatus()) { - String topicName = getTopicPrefix(dbEvent.getHandler().getHiveConf()); + String topicName = getTopicPrefix(dbEvent.getHandler().getConf()); send(messageFactory.buildCreateDatabaseMessage(dbEvent.getDatabase()), topicName); } } @@ -193,7 +195,7 @@ public void onDropDatabase(DropDatabaseEvent dbEvent) throws MetaException { // by listening on a topic named "HCAT" and message selector string // as "HCAT_EVENT = HCAT_DROP_DATABASE" if (dbEvent.getStatus()) { - String topicName = getTopicPrefix(dbEvent.getHandler().getHiveConf()); + String topicName = getTopicPrefix(dbEvent.getHandler().getConf()); send(messageFactory.buildDropDatabaseMessage(dbEvent.getDatabase()), topicName); } } @@ -205,8 +207,8 @@ public void onCreateTable(CreateTableEvent tableEvent) throws MetaException { // as "HCAT_EVENT = HCAT_ADD_TABLE" if (tableEvent.getStatus()) { Table tbl = tableEvent.getTable(); - HMSHandler handler = tableEvent.getHandler(); - HiveConf conf = handler.getHiveConf(); + IHMSHandler handler = tableEvent.getHandler(); + Configuration conf = handler.getConf(); Table newTbl; try { newTbl = handler.get_table_core(tbl.getDbName(), tbl.getTableName()) @@ -216,11 +218,7 @@ public void onCreateTable(CreateTableEvent tableEvent) throws MetaException { getTopicPrefix(conf) + "." + newTbl.getDbName().toLowerCase() + "." + newTbl.getTableName().toLowerCase()); handler.alter_table(newTbl.getDbName(), newTbl.getTableName(), newTbl); - } catch (InvalidOperationException e) { - MetaException me = new MetaException(e.toString()); - me.initCause(e); - throw me; - } catch (NoSuchObjectException e) { + } catch (TException e) { MetaException me = new MetaException(e.toString()); me.initCause(e); throw me; @@ -258,7 +256,7 @@ public void onAlterTable(AlterTableEvent tableEvent) throws MetaException { } // I think this is wrong, the alter table statement should come on the table topic not the // DB topic - Alan. - String topicName = getTopicPrefix(tableEvent.getHandler().getHiveConf()) + "." + + String topicName = getTopicPrefix(tableEvent.getHandler().getConf()) + "." + after.getDbName().toLowerCase(); send(messageFactory.buildAlterTableMessage(before, after), topicName); } @@ -288,7 +286,7 @@ public void onDropTable(DropTableEvent tableEvent) throws MetaException { Table table = tableEvent.getTable(); // I think this is wrong, the drop table statement should come on the table topic not the // DB topic - Alan. - String topicName = getTopicPrefix(tableEvent.getHandler().getHiveConf()) + "." + table.getDbName().toLowerCase(); + String topicName = getTopicPrefix(tableEvent.getHandler().getConf()) + "." + table.getDbName().toLowerCase(); send(messageFactory.buildDropTableMessage(table), topicName); } } diff --git hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java index 3b5066f0f3..49aad392d8 100644 --- hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java +++ hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java @@ -64,6 +64,7 @@ import org.apache.hadoop.hive.metastore.api.TxnInfo; import org.apache.hadoop.hive.metastore.api.TxnState; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; +import org.apache.hadoop.hive.metastore.txn.AcidHouseKeeperService; import org.apache.hadoop.hive.metastore.txn.TxnDbUtil; import org.apache.hadoop.hive.ql.CommandNeedRetryException; import org.apache.hadoop.hive.ql.Driver; @@ -77,7 +78,6 @@ import org.apache.hadoop.hive.ql.io.orc.RecordReader; import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; import org.apache.hadoop.hive.ql.session.SessionState; -import org.apache.hadoop.hive.ql.txn.AcidHouseKeeperService; import org.apache.hadoop.hive.ql.txn.compactor.Worker; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.objectinspector.StructField; @@ -740,11 +740,8 @@ public void testTimeOutReaper() throws Exception { //ensure txn timesout conf.setTimeVar(HiveConf.ConfVars.HIVE_TXN_TIMEOUT, 1, TimeUnit.MILLISECONDS); AcidHouseKeeperService houseKeeperService = new AcidHouseKeeperService(); - houseKeeperService.start(conf); - while(houseKeeperService.getIsAliveCounter() <= Integer.MIN_VALUE) { - Thread.sleep(100);//make sure it has run at least once - } - houseKeeperService.stop(); + houseKeeperService.setConf(conf); + houseKeeperService.run(); try { //should fail because the TransactionBatch timed out txnBatch.commit(); @@ -757,12 +754,7 @@ public void testTimeOutReaper() throws Exception { txnBatch.beginNextTransaction(); txnBatch.commit(); txnBatch.beginNextTransaction(); - int lastCount = houseKeeperService.getIsAliveCounter(); - houseKeeperService.start(conf); - while(houseKeeperService.getIsAliveCounter() <= lastCount) { - Thread.sleep(100);//make sure it has run at least once - } - houseKeeperService.stop(); + houseKeeperService.run(); try { //should fail because the TransactionBatch timed out txnBatch.commit(); diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java index 42e476a8e5..61e3ae696e 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java @@ -509,13 +509,13 @@ public void testMetaConfSameHandler() throws Exception { closingClient.setMetaConf(metaConfKey, "[test pattern modified]"); ConfigChangeEvent event = (ConfigChangeEvent) DummyListener.getLastEvent(); int beforeCloseNotificationEventCounts = DummyListener.notifyList.size(); - HiveMetaStore.HMSHandler beforeHandler = event.getHandler(); + IHMSHandler beforeHandler = event.getHandler(); closingClient.close(); Thread.sleep(5 * 1000); event = (ConfigChangeEvent) DummyListener.getLastEvent(); int afterCloseNotificationEventCounts = DummyListener.notifyList.size(); - HiveMetaStore.HMSHandler afterHandler = event.getHandler(); + IHMSHandler afterHandler = event.getHandler(); // Meta-conf cleanup should trigger an event to listener assertNotSame(beforeCloseNotificationEventCounts, afterCloseNotificationEventCounts); // Both the handlers should be same diff --git itests/util/src/main/java/org/apache/hadoop/hive/ql/security/DummyHiveMetastoreAuthorizationProvider.java itests/util/src/main/java/org/apache/hadoop/hive/ql/security/DummyHiveMetastoreAuthorizationProvider.java index 93637d4321..6662fcd92f 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/ql/security/DummyHiveMetastoreAuthorizationProvider.java +++ itests/util/src/main/java/org/apache/hadoop/hive/ql/security/DummyHiveMetastoreAuthorizationProvider.java @@ -24,7 +24,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.ql.metadata.AuthorizationException; import org.apache.hadoop.hive.ql.metadata.HiveException; @@ -201,7 +201,7 @@ private String debugPrivPrint(Privilege[] privileges) { } @Override - public void setMetaStoreHandler(HMSHandler handler) { + public void setMetaStoreHandler(IHMSHandler handler) { debugLog("DHMAP.setMetaStoreHandler"); } diff --git itests/util/src/main/java/org/apache/hadoop/hive/ql/security/InjectableDummyAuthenticator.java itests/util/src/main/java/org/apache/hadoop/hive/ql/security/InjectableDummyAuthenticator.java index 322834e85e..d4cf026906 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/ql/security/InjectableDummyAuthenticator.java +++ itests/util/src/main/java/org/apache/hadoop/hive/ql/security/InjectableDummyAuthenticator.java @@ -20,7 +20,7 @@ import java.util.List; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.session.SessionState; @@ -91,7 +91,7 @@ public void setConf(Configuration config) { } @Override - public void setMetaStoreHandler(HMSHandler handler) { + public void setMetaStoreHandler(IHMSHandler handler) { hmap.setMetaStoreHandler(handler); } diff --git metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 12faf8202a..339f304933 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -132,7 +132,10 @@ import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge; import org.apache.hadoop.hive.metastore.security.MetastoreDelegationTokenManager; import org.apache.hadoop.hive.metastore.security.TUGIContainingTransport; +import org.apache.hadoop.hive.metastore.txn.AcidHouseKeeperService; import org.apache.hadoop.hive.metastore.txn.AcidOpenTxnsCounterService; +import org.apache.hadoop.hive.metastore.txn.AcidCompactionHistoryService; +import org.apache.hadoop.hive.metastore.txn.AcidWriteSetService; import org.apache.hadoop.hive.metastore.txn.TxnStore; import org.apache.hadoop.hive.metastore.txn.TxnUtils; import org.apache.hadoop.hive.serde2.Deserializer; @@ -219,17 +222,7 @@ public TTransport getTransport(TTransport trans) { } } - /** - * An ugly interface because everything about this file is ugly. RawStore is threadlocal so this - * thread-local disease propagates everywhere, and FileMetadataManager cannot just get a RawStore - * or handlers to use; it will need to have this method to make thread-local handlers and a - * thread-local RawStore. - */ - public interface ThreadLocalRawStore { - RawStore getMS() throws MetaException; - } - - public static class HMSHandler extends FacebookBase implements IHMSHandler, ThreadLocalRawStore { + public static class HMSHandler extends FacebookBase implements IHMSHandler { public static final Logger LOG = HiveMetaStore.LOG; private final HiveConf hiveConf; // stores datastore (jpox) properties, // right now they come from jpox.properties @@ -410,10 +403,20 @@ public static String getThreadLocalIpAddress() { return threadLocalIpAddress.get(); } + /** + * Use {@link #getThreadId()} instead. + * @return thread id + */ + @Deprecated public static Integer get() { return threadLocalId.get(); } + @Override + public int getThreadId() { + return threadLocalId.get(); + } + public HMSHandler(String name) throws MetaException { this(name, new HiveConf(HMSHandler.class), true); } @@ -461,7 +464,8 @@ public HiveConf getHiveConf() { } } - List getTransactionalListeners() { + @Override + public List getTransactionalListeners() { return transactionalListeners; } @@ -541,7 +545,7 @@ public void init() throws MetaException { cleaner.schedule(new DumpDirCleanerTask(hiveConf), cleanFreq, cleanFreq); } expressionProxy = PartFilterExprUtil.createExpressionProxy(hiveConf); - fileMetadataManager = new FileMetadataManager((ThreadLocalRawStore)this, hiveConf); + fileMetadataManager = new FileMetadataManager(this.getMS(), hiveConf); } private static String addPrefix(String s) { @@ -584,6 +588,7 @@ public Configuration getConf() { return modifiedConf; } + @Override public Warehouse getWh() { return wh; } @@ -1031,14 +1036,7 @@ public Database get_database(final String name) throws NoSuchObjectException, Me return db; } - /** - * Equivalent to get_database, but does not write to audit logs, or fire pre-event listners. - * Meant to be used for internal hive classes that don't use the thrift interface. - * @param name - * @return - * @throws NoSuchObjectException - * @throws MetaException - */ + @Override public Database get_database_core(final String name) throws NoSuchObjectException, MetaException { Database db = null; @@ -2404,16 +2402,7 @@ private Table getTableInternal(String dbname, String name, return t; } - /** - * Equivalent of get_table, but does not log audits and fire pre-event listener. - * Meant to be used for calls made by other hive classes, that are not using the - * thrift interface. - * @param dbname - * @param name - * @return Table object - * @throws MetaException - * @throws NoSuchObjectException - */ + @Override public Table get_table_core(final String dbname, final String name) throws MetaException, NoSuchObjectException { Table t = null; @@ -7937,33 +7926,29 @@ private static void startHouseKeeperService(HiveConf conf) throws Exception { if(!HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_COMPACTOR_INITIATOR_ON)) { return; } - startHouseKeeperService(conf, Class.forName("org.apache.hadoop.hive.ql.txn.AcidHouseKeeperService")); - startHouseKeeperService(conf, Class.forName("org.apache.hadoop.hive.ql.txn.AcidCompactionHistoryService")); - startHouseKeeperService(conf, Class.forName("org.apache.hadoop.hive.ql.txn.AcidWriteSetService")); ThreadPool.initialize(conf); - RunnableConfigurable rc = new AcidOpenTxnsCounterService(); - rc.setConf(conf); - ThreadPool.getPool().scheduleAtFixedRate(rc, 100, MetastoreConf.getTimeVar(conf, - MetastoreConf.ConfVars.COUNT_OPEN_TXNS_INTERVAL, TimeUnit.MILLISECONDS), - TimeUnit.MILLISECONDS); - + startOneHouseKeeperService(new AcidHouseKeeperService(), conf, + MetastoreConf.getTimeVar(conf, MetastoreConf.ConfVars.TIMEDOUT_TXN_REAPER_INTERVAL, + TimeUnit.MILLISECONDS)); + startOneHouseKeeperService(new AcidOpenTxnsCounterService(), conf, + MetastoreConf.getTimeVar(conf, MetastoreConf.ConfVars.COUNT_OPEN_TXNS_INTERVAL, + TimeUnit.MILLISECONDS)); + startOneHouseKeeperService(new AcidCompactionHistoryService(), conf, + MetastoreConf.getTimeVar(conf, MetastoreConf.ConfVars.COMPACTOR_HISTORY_REAPER_INTERVAL, + TimeUnit.MILLISECONDS)); + startOneHouseKeeperService(new AcidWriteSetService(), conf, + MetastoreConf.getTimeVar(conf, MetastoreConf.ConfVars.WRITE_SET_REAPER_INTERVAL, + TimeUnit.MILLISECONDS)); } - private static void startHouseKeeperService(HiveConf conf, Class c) throws Exception { - //todo: when metastore adds orderly-shutdown logic, houseKeeper.stop() - //should be called form it - HouseKeeperService houseKeeper = (HouseKeeperService)c.newInstance(); - try { - houseKeeper.start(conf); - } - catch (Exception ex) { - LOG.error("Failed to start {}" , houseKeeper.getClass() + - ". The system will not handle {} " , houseKeeper.getServiceDescription(), - ". Root Cause: ", ex); - } + + private static void startOneHouseKeeperService(RunnableConfigurable rc, Configuration conf, + long interval) { + rc.setConf(conf); + ThreadPool.getPool().scheduleAtFixedRate(rc, 0, interval, TimeUnit.MILLISECONDS); } - public static Map createHandlerMap() { + static Map createHandlerMap() { Map fmHandlers = new HashMap<>(); for (FileMetadataExprType v : FileMetadataExprType.values()) { switch (v) { diff --git metastore/src/java/org/apache/hadoop/hive/metastore/HouseKeeperService.java metastore/src/java/org/apache/hadoop/hive/metastore/HouseKeeperService.java deleted file mode 100644 index 539ace0067..0000000000 --- metastore/src/java/org/apache/hadoop/hive/metastore/HouseKeeperService.java +++ /dev/null @@ -1,45 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.metastore; - -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.conf.HiveConf; - -/** - * Runs arbitrary background logic inside the metastore service. - */ -@InterfaceAudience.LimitedPrivate({"Hive"}) -@InterfaceStability.Evolving -public interface HouseKeeperService { - public void start(HiveConf hiveConf) throws Exception; - /** - * Should perform orderly shutdown - */ - public void stop(); - /** - * Returns short description of services this module provides. - */ - public String getServiceDescription(); - - /** - * This is incremented each time the service is performed. Can be useful to - * check if serivce is still alive. - */ - public int getIsAliveCounter(); -} diff --git metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java index 5354e704d8..4e47306395 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java @@ -22,13 +22,8 @@ import java.io.IOException; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; -import java.math.BigDecimal; -import java.math.BigInteger; import java.net.URL; import java.net.URLClassLoader; -import java.nio.charset.Charset; -import java.nio.charset.StandardCharsets; -import java.security.MessageDigest; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -40,29 +35,11 @@ import java.util.Map.Entry; import java.util.Properties; import java.util.Set; -import java.util.SortedMap; -import java.util.SortedSet; -import java.util.TreeMap; -import java.util.TreeSet; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; import java.util.regex.Matcher; import java.util.regex.Pattern; -import com.google.common.base.Predicates; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.google.common.util.concurrent.ThreadFactoryBuilder; - import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.collections.ListUtils; import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.hive.metastore.api.Decimal; -import org.apache.hadoop.hive.metastore.api.Order; -import org.apache.hadoop.hive.metastore.api.SkewedInfo; import org.apache.hadoop.hive.shims.ShimLoader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -81,15 +58,12 @@ import org.apache.hadoop.hive.metastore.api.EnvironmentContext; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.InvalidObjectException; -import org.apache.hadoop.hive.metastore.api.InvalidOperationException; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.SerDeInfo; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; -import org.apache.hadoop.hive.metastore.columnstats.aggr.ColumnStatsAggregator; -import org.apache.hadoop.hive.metastore.columnstats.aggr.ColumnStatsAggregatorFactory; import org.apache.hadoop.hive.metastore.columnstats.merge.ColumnStatsMerger; import org.apache.hadoop.hive.metastore.columnstats.merge.ColumnStatsMergerFactory; import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy; @@ -194,7 +168,7 @@ static public void recursiveDelete(File f) throws IOException { * @param partParams * @return True if the passed Parameters Map contains values for all "Fast Stats". */ - public static boolean containsAllFastStats(Map partParams) { + private static boolean containsAllFastStats(Map partParams) { for (String stat : StatsSetupConst.fastStats) { if (!partParams.containsKey(stat)) { return false; @@ -203,12 +177,12 @@ public static boolean containsAllFastStats(Map partParams) { return true; } - public static boolean updateTableStatsFast(Database db, Table tbl, Warehouse wh, + static boolean updateTableStatsFast(Database db, Table tbl, Warehouse wh, boolean madeDir, EnvironmentContext environmentContext) throws MetaException { return updateTableStatsFast(db, tbl, wh, madeDir, false, environmentContext); } - public static boolean updateTableStatsFast(Database db, Table tbl, Warehouse wh, + private static boolean updateTableStatsFast(Database db, Table tbl, Warehouse wh, boolean madeDir, boolean forceRecompute, EnvironmentContext environmentContext) throws MetaException { if (tbl.getPartitionKeysSize() == 0) { // Update stats only when unpartitioned @@ -288,70 +262,12 @@ public static void populateQuickStats(FileStatus[] fileStatus, Map params = part.getParameters(); boolean updated = false; @@ -407,7 +323,8 @@ public static boolean updatePartitionStatsFast(PartitionSpecProxy.PartitionItera return updated; } - static void updateBasicState(EnvironmentContext environmentContext, Map params) { + private static void updateBasicState(EnvironmentContext environmentContext, Map + params) { if (params == null) { return; } @@ -612,7 +529,7 @@ static public boolean validateName(String name, Configuration conf) { /* * At the Metadata level there are no restrictions on Column Names. */ - public static final boolean validateColumnName(String name) { + public static boolean validateColumnName(String name) { return true; } @@ -629,53 +546,6 @@ static public String validateTblColumns(List cols) { return null; } - static void throwExceptionIfIncompatibleColTypeChange( - List oldCols, List newCols) - throws InvalidOperationException { - - List incompatibleCols = new ArrayList(); - int maxCols = Math.min(oldCols.size(), newCols.size()); - for (int i = 0; i < maxCols; i++) { - if (!areColTypesCompatible(oldCols.get(i).getType(), newCols.get(i).getType())) { - incompatibleCols.add(newCols.get(i).getName()); - } - } - if (!incompatibleCols.isEmpty()) { - throw new InvalidOperationException( - "The following columns have types incompatible with the existing " + - "columns in their respective positions :\n" + - StringUtils.join(incompatibleCols, ',') - ); - } - } - - static boolean areSameColumns(List oldCols, List newCols) { - return ListUtils.isEqualList(oldCols, newCols); - } - - /* - * This method is to check if the new column list includes all the old columns with same name and - * type. The column comment does not count. - */ - static boolean columnsIncludedByNameType(List oldCols, List newCols) { - if (oldCols.size() > newCols.size()) { - return false; - } - - Map columnNameTypePairMap = new HashMap(newCols.size()); - for (FieldSchema newCol : newCols) { - columnNameTypePairMap.put(newCol.getName().toLowerCase(), newCol.getType()); - } - for (final FieldSchema oldCol : oldCols) { - if (!columnNameTypePairMap.containsKey(oldCol.getName()) - || !columnNameTypePairMap.get(oldCol.getName()).equalsIgnoreCase(oldCol.getType())) { - return false; - } - } - - return true; - } - /** * @return true if oldType and newType are compatible. * Two types are compatible if we have internal functions to cast one to another. @@ -821,7 +691,7 @@ public static void setSerdeParam(SerDeInfo sdi, Properties schema, org.apache.hadoop.hive.serde.serdeConstants.INTERVAL_DAY_TIME_TYPE_NAME); } - static Set hiveThriftTypeMap; //for validation + private static Set hiveThriftTypeMap; //for validation static { hiveThriftTypeMap = new HashSet(); hiveThriftTypeMap.addAll(serdeConstants.PrimitiveTypes); @@ -1676,31 +1546,6 @@ public static int getArchivingLevel(Partition part) throws MetaException { }; /** - * We have aneed to sanity-check the map before conversion from persisted objects to - * metadata thrift objects because null values in maps will cause a NPE if we send - * across thrift. Pruning is appropriate for most cases except for databases such as - * Oracle where Empty strings are stored as nulls, in which case we need to handle that. - * See HIVE-8485 for motivations for this. - */ - public static Map trimMapNulls( - Map dnMap, boolean retrieveMapNullsAsEmptyStrings){ - if (dnMap == null){ - return null; - } - // Must be deterministic order map - see HIVE-8707 - // => we use Maps.newLinkedHashMap instead of Maps.newHashMap - if (retrieveMapNullsAsEmptyStrings) { - // convert any nulls present in map values to empty strings - this is done in the case - // of backing dbs like oracle which persist empty strings as nulls. - return Maps.newLinkedHashMap(Maps.transformValues(dnMap, transFormNullsToEmptyString)); - } else { - // prune any nulls present in map values - this is the typical case. - return Maps.newLinkedHashMap(Maps.filterValues(dnMap, Predicates.notNull())); - } - } - - - /** * Create a URL from a string representing a path to a local file. * The path string can be just a path, or can start with file:/, file:/// * @param onestr path string @@ -1780,20 +1625,6 @@ public static void mergeColStats(ColumnStatistics csNew, ColumnStatistics csOld) csNew.setStatsObj(list); } - /** - * convert Exception to MetaException, which sets the cause to such exception - * @param errorMessage the error message for this MetaException - * @param e cause of the exception - * @return the MetaException with the specified exception as the cause - */ - public static MetaException newMetaException(String errorMessage, Exception e) { - MetaException metaException = new MetaException(errorMessage); - if (e != null) { - metaException.initCause(e); - } - return metaException; - } - public static List getColumnNames(List schema) { List cols = new ArrayList<>(schema.size()); for (FieldSchema fs : schema) { diff --git metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreUtils.java metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreUtils.java deleted file mode 100644 index e5c8a40e39..0000000000 --- metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreUtils.java +++ /dev/null @@ -1,43 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.metastore; - -import org.apache.hadoop.hive.metastore.api.FieldSchema; -import org.junit.Assert; -import org.junit.Test; - -import java.util.Arrays; - -public class TestMetaStoreUtils { - - @Test - public void testcolumnsIncludedByNameType() { - FieldSchema col1 = new FieldSchema("col1", "string", "col1 comment"); - FieldSchema col1a = new FieldSchema("col1", "string", "col1 but with a different comment"); - FieldSchema col2 = new FieldSchema("col2", "string", "col2 comment"); - FieldSchema col3 = new FieldSchema("col3", "string", "col3 comment"); - Assert.assertTrue(MetaStoreUtils.columnsIncludedByNameType(Arrays.asList(col1), Arrays.asList(col1))); - Assert.assertTrue(MetaStoreUtils.columnsIncludedByNameType(Arrays.asList(col1), Arrays.asList(col1a))); - Assert.assertTrue(MetaStoreUtils.columnsIncludedByNameType(Arrays.asList(col1, col2), Arrays.asList(col1, col2))); - Assert.assertTrue(MetaStoreUtils.columnsIncludedByNameType(Arrays.asList(col1, col2), Arrays.asList(col2, col1))); - Assert.assertTrue(MetaStoreUtils.columnsIncludedByNameType(Arrays.asList(col1, col2), Arrays.asList(col1, col2, col3))); - Assert.assertTrue(MetaStoreUtils.columnsIncludedByNameType(Arrays.asList(col1, col2), Arrays.asList(col3, col2, col1))); - Assert.assertFalse(MetaStoreUtils.columnsIncludedByNameType(Arrays.asList(col1, col2), Arrays.asList(col1))); - } -} diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/MetaDataExportListener.java ql/src/java/org/apache/hadoop/hive/ql/parse/MetaDataExportListener.java index db624c0cb5..7a28b433af 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/MetaDataExportListener.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/MetaDataExportListener.java @@ -21,15 +21,15 @@ import java.text.SimpleDateFormat; import java.util.Date; +import org.apache.hadoop.hive.metastore.IHMSHandler; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.MetaStorePreEventListener; import org.apache.hadoop.hive.metastore.Warehouse; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; import org.apache.hadoop.hive.metastore.api.InvalidOperationException; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; @@ -58,17 +58,17 @@ private void export_meta_data(PreDropTableEvent tableEvent) throws MetaException String name = tbl.getTableName(); org.apache.hadoop.hive.ql.metadata.Table mTbl = new org.apache.hadoop.hive.ql.metadata.Table( tbl); - HMSHandler handler = tableEvent.getHandler(); - HiveConf hiveconf = handler.getHiveConf(); - Warehouse wh = new Warehouse(hiveconf); + IHMSHandler handler = tableEvent.getHandler(); + Configuration conf = handler.getConf(); + Warehouse wh = new Warehouse(conf); Path tblPath = new Path(tbl.getSd().getLocation()); fs = wh.getFs(tblPath); Date now = new Date(); SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss"); String dateString = sdf.format(now); - String exportPathString = hiveconf.getVar(HiveConf.ConfVars.METADATA_EXPORT_LOCATION); - boolean moveMetadataToTrash = hiveconf - .getBoolVar(HiveConf.ConfVars.MOVE_EXPORTED_METADATA_TO_TRASH); + String exportPathString = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.METADATA_EXPORT_LOCATION); + boolean moveMetadataToTrash = MetastoreConf + .getBoolVar(conf, MetastoreConf.ConfVars.MOVE_EXPORTED_METADATA_TO_TRASH); Path exportPath = null; if (exportPathString != null && exportPathString.length() == 0) { exportPath = fs.getHomeDirectory(); diff --git ql/src/java/org/apache/hadoop/hive/ql/security/HadoopDefaultMetastoreAuthenticator.java ql/src/java/org/apache/hadoop/hive/ql/security/HadoopDefaultMetastoreAuthenticator.java index c3720270fe..07b506eb53 100644 --- ql/src/java/org/apache/hadoop/hive/ql/security/HadoopDefaultMetastoreAuthenticator.java +++ ql/src/java/org/apache/hadoop/hive/ql/security/HadoopDefaultMetastoreAuthenticator.java @@ -18,13 +18,13 @@ package org.apache.hadoop.hive.ql.security; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.hive.metastore.IHMSHandler; public class HadoopDefaultMetastoreAuthenticator extends HadoopDefaultAuthenticator implements HiveMetastoreAuthenticationProvider { @Override - public void setMetaStoreHandler(HMSHandler handler) { + public void setMetaStoreHandler(IHMSHandler handler) { setConf(handler.getConf()); } diff --git ql/src/java/org/apache/hadoop/hive/ql/security/HiveMetastoreAuthenticationProvider.java ql/src/java/org/apache/hadoop/hive/ql/security/HiveMetastoreAuthenticationProvider.java index 73e9bfb61b..9b83c23b5f 100644 --- ql/src/java/org/apache/hadoop/hive/ql/security/HiveMetastoreAuthenticationProvider.java +++ ql/src/java/org/apache/hadoop/hive/ql/security/HiveMetastoreAuthenticationProvider.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hive.ql.security; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.hive.metastore.IHMSHandler; /** @@ -36,6 +36,6 @@ * authentication that needs to be done. * @param handler */ - void setMetaStoreHandler(HMSHandler handler); + void setMetaStoreHandler(IHMSHandler handler); } diff --git ql/src/java/org/apache/hadoop/hive/ql/security/authorization/DefaultHiveMetastoreAuthorizationProvider.java ql/src/java/org/apache/hadoop/hive/ql/security/authorization/DefaultHiveMetastoreAuthorizationProvider.java index 41202988db..d3242e3e48 100644 --- ql/src/java/org/apache/hadoop/hive/ql/security/authorization/DefaultHiveMetastoreAuthorizationProvider.java +++ ql/src/java/org/apache/hadoop/hive/ql/security/authorization/DefaultHiveMetastoreAuthorizationProvider.java @@ -19,7 +19,7 @@ package org.apache.hadoop.hive.ql.security.authorization; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.ql.metadata.AuthorizationException; import org.apache.hadoop.hive.ql.metadata.HiveException; @@ -32,7 +32,7 @@ public void init(Configuration conf) throws HiveException { } @Override - public void setMetaStoreHandler(HMSHandler handler) { + public void setMetaStoreHandler(IHMSHandler handler) { hive_db.setHandler(handler); } diff --git ql/src/java/org/apache/hadoop/hive/ql/security/authorization/HiveAuthorizationProviderBase.java ql/src/java/org/apache/hadoop/hive/ql/security/authorization/HiveAuthorizationProviderBase.java index 191426c930..fa66a21c57 100644 --- ql/src/java/org/apache/hadoop/hive/ql/security/authorization/HiveAuthorizationProviderBase.java +++ ql/src/java/org/apache/hadoop/hive/ql/security/authorization/HiveAuthorizationProviderBase.java @@ -20,11 +20,11 @@ import java.util.List; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.HiveObjectRef; import org.apache.hadoop.hive.metastore.api.HiveObjectType; @@ -43,7 +43,7 @@ private final boolean hasHiveClient; private final HiveConf conf; - private HMSHandler handler; + private IHMSHandler handler; public HiveProxy(Hive hive) { this.hasHiveClient = hive != null; @@ -57,7 +57,7 @@ public HiveProxy() { this.handler = null; } - public void setHandler(HMSHandler handler){ + public void setHandler(IHMSHandler handler){ this.handler = handler; } diff --git ql/src/java/org/apache/hadoop/hive/ql/security/authorization/HiveMetastoreAuthorizationProvider.java ql/src/java/org/apache/hadoop/hive/ql/security/authorization/HiveMetastoreAuthorizationProvider.java index 23161e3be6..bf21de7e49 100644 --- ql/src/java/org/apache/hadoop/hive/ql/security/authorization/HiveMetastoreAuthorizationProvider.java +++ ql/src/java/org/apache/hadoop/hive/ql/security/authorization/HiveMetastoreAuthorizationProvider.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hive.ql.security.authorization; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.ql.metadata.AuthorizationException; import org.apache.hadoop.hive.ql.metadata.HiveException; @@ -37,7 +37,7 @@ * before any of the authorize methods are called. * @param handler */ - void setMetaStoreHandler(HMSHandler handler); + void setMetaStoreHandler(IHMSHandler handler); /** * Authorize metastore authorization api call. diff --git ql/src/java/org/apache/hadoop/hive/ql/security/authorization/MetaStoreAuthzAPIAuthorizerEmbedOnly.java ql/src/java/org/apache/hadoop/hive/ql/security/authorization/MetaStoreAuthzAPIAuthorizerEmbedOnly.java index 5f1725c4a2..00d95ef913 100644 --- ql/src/java/org/apache/hadoop/hive/ql/security/authorization/MetaStoreAuthzAPIAuthorizerEmbedOnly.java +++ ql/src/java/org/apache/hadoop/hive/ql/security/authorization/MetaStoreAuthzAPIAuthorizerEmbedOnly.java @@ -22,7 +22,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.HiveMetaStore; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.ql.metadata.AuthorizationException; import org.apache.hadoop.hive.ql.metadata.HiveException; @@ -76,7 +76,7 @@ public void authorize(Table table, Partition part, List columns, } @Override - public void setMetaStoreHandler(HMSHandler handler) { + public void setMetaStoreHandler(IHMSHandler handler) { // no-op - HMSHander not needed by this impl } diff --git ql/src/java/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java ql/src/java/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java index 32d8b3eb66..56fc9e46d2 100644 --- ql/src/java/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java +++ ql/src/java/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java @@ -27,6 +27,7 @@ import javax.security.auth.login.LoginException; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; @@ -36,7 +37,6 @@ import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.Database; @@ -271,7 +271,7 @@ public void authorize(Table table, Partition part, List columns, } @Override - public void setMetaStoreHandler(HMSHandler handler) { + public void setMetaStoreHandler(IHMSHandler handler) { hive_db.setHandler(handler); this.wh = handler.getWh(); this.isRunFromMetaStore = true; diff --git ql/src/java/org/apache/hadoop/hive/ql/txn/AcidCompactionHistoryService.java ql/src/java/org/apache/hadoop/hive/ql/txn/AcidCompactionHistoryService.java deleted file mode 100644 index 5d9e7bea90..0000000000 --- ql/src/java/org/apache/hadoop/hive/ql/txn/AcidCompactionHistoryService.java +++ /dev/null @@ -1,82 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.ql.txn; - -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.txn.TxnStore; -import org.apache.hadoop.hive.metastore.txn.TxnUtils; -import org.apache.hadoop.hive.ql.txn.compactor.HouseKeeperServiceBase; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * Purges obsolete items from compaction history data - */ -public class AcidCompactionHistoryService extends HouseKeeperServiceBase { - private static final Logger LOG = LoggerFactory.getLogger(AcidCompactionHistoryService.class); - - @Override - protected long getStartDelayMs() { - return 0; - } - @Override - protected long getIntervalMs() { - return hiveConf.getTimeVar(HiveConf.ConfVars.COMPACTOR_HISTORY_REAPER_INTERVAL, TimeUnit.MILLISECONDS); - } - @Override - protected Runnable getScheduedAction(HiveConf hiveConf, AtomicInteger isAliveCounter) { - return new ObsoleteEntryReaper(hiveConf, isAliveCounter); - } - - @Override - public String getServiceDescription() { - return "Removes obsolete entries from Compaction History"; - } - - private static final class ObsoleteEntryReaper implements Runnable { - private final TxnStore txnHandler; - private final AtomicInteger isAliveCounter; - private ObsoleteEntryReaper(HiveConf hiveConf, AtomicInteger isAliveCounter) { - txnHandler = TxnUtils.getTxnStore(hiveConf); - this.isAliveCounter = isAliveCounter; - } - - @Override - public void run() { - TxnStore.MutexAPI.LockHandle handle = null; - try { - handle = txnHandler.getMutexAPI().acquireLock(TxnStore.MUTEX_KEY.CompactionHistory.name()); - long startTime = System.currentTimeMillis(); - txnHandler.purgeCompactionHistory(); - int count = isAliveCounter.incrementAndGet(); - LOG.info("History reaper reaper ran for " + (System.currentTimeMillis() - startTime)/1000 + "seconds. isAliveCounter=" + count); - } - catch(Throwable t) { - LOG.error("Serious error in {}", Thread.currentThread().getName(), ": {}" + t.getMessage(), t); - } - finally { - if(handle != null) { - handle.releaseLocks(); - } - } - } - } -} diff --git ql/src/java/org/apache/hadoop/hive/ql/txn/AcidHouseKeeperService.java ql/src/java/org/apache/hadoop/hive/ql/txn/AcidHouseKeeperService.java deleted file mode 100644 index 13b10de588..0000000000 --- ql/src/java/org/apache/hadoop/hive/ql/txn/AcidHouseKeeperService.java +++ /dev/null @@ -1,82 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.ql.txn; - -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.txn.TxnStore; -import org.apache.hadoop.hive.metastore.txn.TxnUtils; -import org.apache.hadoop.hive.ql.txn.compactor.HouseKeeperServiceBase; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * Performs background tasks for Transaction management in Hive. - * Runs inside Hive Metastore Service. - */ -public class AcidHouseKeeperService extends HouseKeeperServiceBase { - private static final Logger LOG = LoggerFactory.getLogger(AcidHouseKeeperService.class); - - @Override - protected long getStartDelayMs() { - return hiveConf.getTimeVar(HiveConf.ConfVars.HIVE_TIMEDOUT_TXN_REAPER_START, TimeUnit.MILLISECONDS); - } - @Override - protected long getIntervalMs() { - return hiveConf.getTimeVar(HiveConf.ConfVars.HIVE_TIMEDOUT_TXN_REAPER_INTERVAL, TimeUnit.MILLISECONDS); - } - @Override - protected Runnable getScheduedAction(HiveConf hiveConf, AtomicInteger isAliveCounter) { - return new TimedoutTxnReaper(hiveConf, isAliveCounter); - } - - @Override - public String getServiceDescription() { - return "Abort expired transactions"; - } - - private static final class TimedoutTxnReaper implements Runnable { - private final TxnStore txnHandler; - private final AtomicInteger isAliveCounter; - private TimedoutTxnReaper(HiveConf hiveConf, AtomicInteger isAliveCounter) { - txnHandler = TxnUtils.getTxnStore(hiveConf); - this.isAliveCounter = isAliveCounter; - } - @Override - public void run() { - TxnStore.MutexAPI.LockHandle handle = null; - try { - handle = txnHandler.getMutexAPI().acquireLock(TxnStore.MUTEX_KEY.HouseKeeper.name()); - long startTime = System.currentTimeMillis(); - txnHandler.performTimeOuts(); - int count = isAliveCounter.incrementAndGet(); - LOG.info("timeout reaper ran for " + (System.currentTimeMillis() - startTime)/1000 + "seconds. isAliveCounter=" + count); - } - catch(Throwable t) { - LOG.error("Serious error in {}", Thread.currentThread().getName(), ": {}" + t.getMessage(), t); - } - finally { - if(handle != null) { - handle.releaseLocks(); - } - } - } - } -} diff --git ql/src/java/org/apache/hadoop/hive/ql/txn/AcidWriteSetService.java ql/src/java/org/apache/hadoop/hive/ql/txn/AcidWriteSetService.java deleted file mode 100644 index 2b8f8154ab..0000000000 --- ql/src/java/org/apache/hadoop/hive/ql/txn/AcidWriteSetService.java +++ /dev/null @@ -1,78 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.ql.txn; - -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.txn.TxnStore; -import org.apache.hadoop.hive.metastore.txn.TxnUtils; -import org.apache.hadoop.hive.ql.txn.compactor.HouseKeeperServiceBase; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * Periodically cleans WriteSet tracking information used in Transaction management - */ -public class AcidWriteSetService extends HouseKeeperServiceBase { - private static final Logger LOG = LoggerFactory.getLogger(AcidWriteSetService.class); - @Override - protected long getStartDelayMs() { - return 0; - } - @Override - protected long getIntervalMs() { - return hiveConf.getTimeVar(HiveConf.ConfVars.WRITE_SET_REAPER_INTERVAL, TimeUnit.MILLISECONDS); - } - @Override - protected Runnable getScheduedAction(HiveConf hiveConf, AtomicInteger isAliveCounter) { - return new WriteSetReaper(hiveConf, isAliveCounter); - } - @Override - public String getServiceDescription() { - return "Periodically cleans obsolete WriteSet tracking information used in Transaction management"; - } - private static final class WriteSetReaper implements Runnable { - private final TxnStore txnHandler; - private final AtomicInteger isAliveCounter; - private WriteSetReaper(HiveConf hiveConf, AtomicInteger isAliveCounter) { - txnHandler = TxnUtils.getTxnStore(hiveConf); - this.isAliveCounter = isAliveCounter; - } - @Override - public void run() { - TxnStore.MutexAPI.LockHandle handle = null; - try { - handle = txnHandler.getMutexAPI().acquireLock(TxnStore.MUTEX_KEY.WriteSetCleaner.name()); - long startTime = System.currentTimeMillis(); - txnHandler.performWriteSetGC(); - int count = isAliveCounter.incrementAndGet(); - LOG.info("cleaner ran for " + (System.currentTimeMillis() - startTime)/1000 + "seconds. isAliveCounter=" + count); - } - catch(Throwable t) { - LOG.error("Serious error in {}", Thread.currentThread().getName(), ": {}" + t.getMessage(), t); - } - finally { - if(handle != null) { - handle.releaseLocks(); - } - } - } - } -} diff --git ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/HouseKeeperServiceBase.java ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/HouseKeeperServiceBase.java deleted file mode 100644 index 0aa160cb39..0000000000 --- ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/HouseKeeperServiceBase.java +++ /dev/null @@ -1,96 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */package org.apache.hadoop.hive.ql.txn.compactor; - -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.HouseKeeperService; -import org.apache.hadoop.hive.ql.lockmgr.HiveTxnManager; -import org.apache.hadoop.hive.ql.lockmgr.TxnManagerFactory; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; - -public abstract class HouseKeeperServiceBase implements HouseKeeperService { - private static final Logger LOG = LoggerFactory.getLogger(HouseKeeperServiceBase.class); - private ScheduledExecutorService pool = null; - protected final AtomicInteger isAliveCounter = new AtomicInteger(Integer.MIN_VALUE); - protected HiveConf hiveConf; - - @Override - public void start(HiveConf hiveConf) throws Exception { - this.hiveConf = hiveConf; - HiveTxnManager mgr = TxnManagerFactory.getTxnManagerFactory().getTxnManager(hiveConf); - if(!mgr.supportsAcid()) { - LOG.info(this.getClass().getName() + " not started since " + - mgr.getClass().getName() + " does not support Acid."); - return;//there are no transactions in this case - } - pool = Executors.newScheduledThreadPool(1, new ThreadFactory() { - private final AtomicInteger threadCounter = new AtomicInteger(); - @Override - public Thread newThread(Runnable r) { - Thread t = - new Thread(r, HouseKeeperServiceBase.this.getClass().getName() + "-" - + threadCounter.getAndIncrement()); - t.setDaemon(true); - return t; - } - }); - - TimeUnit tu = TimeUnit.MILLISECONDS; - pool.scheduleAtFixedRate(getScheduedAction(hiveConf, isAliveCounter), getStartDelayMs(), - getIntervalMs(), tu); - LOG.info("Started " + this.getClass().getName() + " with delay/interval = " + getStartDelayMs() + "/" + - getIntervalMs() + " " + tu); - } - - @Override - public void stop() { - if(pool != null && !pool.isShutdown()) { - pool.shutdown(); - } - pool = null; - } - - /** - * This is used for testing only. Each time the housekeeper runs, counter is incremented by 1. - * Starts with {@link java.lang.Integer#MIN_VALUE} - */ - @Override - public int getIsAliveCounter() { - return isAliveCounter.get(); - } - - /** - * Delay in millis before first run of the task of this service. - */ - protected abstract long getStartDelayMs(); - /** - * Determines how frequently the service is running its task. - */ - protected abstract long getIntervalMs(); - - /** - * The actual task implementation. Must increment the counter on each iteration. - */ - protected abstract Runnable getScheduedAction(HiveConf hiveConf, AtomicInteger isAliveCounter); -} diff --git ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java index 39d6b2b414..662462c005 100644 --- ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java +++ ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java @@ -17,9 +17,9 @@ */ package org.apache.hadoop.hive.ql; -import org.apache.commons.io.FileUtils; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.RunnableConfigurable; import org.apache.hadoop.hive.metastore.api.GetOpenTxnsInfoResponse; import org.apache.hadoop.hive.metastore.api.LockState; import org.apache.hadoop.hive.metastore.api.LockType; @@ -27,6 +27,7 @@ import org.apache.hadoop.hive.metastore.api.ShowLocksResponse; import org.apache.hadoop.hive.metastore.api.TxnInfo; import org.apache.hadoop.hive.metastore.api.TxnState; +import org.apache.hadoop.hive.metastore.txn.AcidHouseKeeperService; import org.apache.hadoop.hive.metastore.txn.TxnDbUtil; import org.apache.hadoop.hive.metastore.txn.TxnStore; import org.apache.hadoop.hive.metastore.txn.TxnUtils; @@ -36,7 +37,6 @@ import org.apache.hadoop.hive.ql.lockmgr.TestDbTxnManager2; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; -import org.apache.hadoop.hive.ql.txn.AcidHouseKeeperService; import org.junit.After; import org.junit.Assert; import org.junit.Ignore; @@ -337,11 +337,11 @@ public void testTimeOutReaper() throws Exception { runStatementOnDriver("start transaction"); runStatementOnDriver("delete from " + Table.ACIDTBL + " where a = 5"); //make sure currently running txn is considered aborted by housekeeper - hiveConf.setTimeVar(HiveConf.ConfVars.HIVE_TIMEDOUT_TXN_REAPER_START, 0, TimeUnit.SECONDS); hiveConf.setTimeVar(HiveConf.ConfVars.HIVE_TXN_TIMEOUT, 2, TimeUnit.MILLISECONDS); - AcidHouseKeeperService houseKeeperService = new AcidHouseKeeperService(); + RunnableConfigurable houseKeeperService = new AcidHouseKeeperService(); + houseKeeperService.setConf(hiveConf); //this will abort the txn - TestTxnCommands2.runHouseKeeperService(houseKeeperService, hiveConf); + houseKeeperService.run(); //this should fail because txn aborted due to timeout CommandProcessorResponse cpr = runStatementOnDriverNegative("delete from " + Table.ACIDTBL + " where a = 5"); Assert.assertTrue("Actual: " + cpr.getErrorMessage(), cpr.getErrorMessage().contains("Transaction manager has aborted the transaction txnid:1")); @@ -349,6 +349,8 @@ public void testTimeOutReaper() throws Exception { //now test that we don't timeout locks we should not //heartbeater should be running in the background every 1/2 second hiveConf.setTimeVar(HiveConf.ConfVars.HIVE_TXN_TIMEOUT, 1, TimeUnit.SECONDS); + // Have to reset the conf when we change it so that the change takes affect + houseKeeperService.setConf(hiveConf); //hiveConf.setBoolVar(HiveConf.ConfVars.HIVETESTMODEFAILHEARTBEATER, true); runStatementOnDriver("start transaction"); runStatementOnDriver("select count(*) from " + Table.ACIDTBL + " where a = 17"); @@ -380,14 +382,14 @@ public void testTimeOutReaper() throws Exception { ShowLocksResponse slr = txnHandler.showLocks(new ShowLocksRequest()); TestDbTxnManager2.checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", Table.ACIDTBL.name, null, slr.getLocks()); pause(750); - TestTxnCommands2.runHouseKeeperService(houseKeeperService, hiveConf); + houseKeeperService.run(); pause(750); slr = txnHandler.showLocks(new ShowLocksRequest()); Assert.assertEquals("Unexpected lock count: " + slr, 1, slr.getLocks().size()); TestDbTxnManager2.checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", Table.ACIDTBL.name, null, slr.getLocks()); pause(750); - TestTxnCommands2.runHouseKeeperService(houseKeeperService, hiveConf); + houseKeeperService.run(); slr = txnHandler.showLocks(new ShowLocksRequest()); Assert.assertEquals("Unexpected lock count: " + slr, 1, slr.getLocks().size()); TestDbTxnManager2.checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", Table.ACIDTBL.name, null, slr.getLocks()); diff --git ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java index 3737b6a4d5..2faf098f6a 100644 --- ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java +++ ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java @@ -36,7 +36,6 @@ import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.common.ValidTxnList; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.HouseKeeperService; import org.apache.hadoop.hive.metastore.api.CommitTxnRequest; import org.apache.hadoop.hive.metastore.api.CompactionRequest; import org.apache.hadoop.hive.metastore.api.CompactionType; @@ -46,6 +45,7 @@ import org.apache.hadoop.hive.metastore.api.ShowCompactRequest; import org.apache.hadoop.hive.metastore.api.ShowCompactResponse; import org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement; +import org.apache.hadoop.hive.metastore.txn.AcidCompactionHistoryService; import org.apache.hadoop.hive.metastore.txn.TxnDbUtil; import org.apache.hadoop.hive.metastore.txn.TxnStore; import org.apache.hadoop.hive.metastore.txn.TxnUtils; @@ -54,7 +54,6 @@ import org.apache.hadoop.hive.ql.io.HiveInputFormat; import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; import org.apache.hadoop.hive.ql.session.SessionState; -import org.apache.hadoop.hive.ql.txn.AcidCompactionHistoryService; import org.apache.hadoop.hive.metastore.txn.AcidOpenTxnsCounterService; import org.apache.hadoop.hive.ql.txn.compactor.Cleaner; import org.apache.hadoop.hive.ql.txn.compactor.Initiator; @@ -1041,7 +1040,8 @@ void testInitiatorWithMultipleFailedCompactionsForVariousTblProperties(String tb hiveConf.setTimeVar(HiveConf.ConfVars.COMPACTOR_HISTORY_REAPER_INTERVAL, 10, TimeUnit.MILLISECONDS); AcidCompactionHistoryService compactionHistoryService = new AcidCompactionHistoryService(); - runHouseKeeperService(compactionHistoryService, hiveConf);//should not remove anything from history + compactionHistoryService.setConf(hiveConf); + compactionHistoryService.run(); checkCompactionState(new CompactionsByState(numAttemptedCompactions,numFailedCompactions,0,0,0,0,numFailedCompactions + numAttemptedCompactions), countCompacts(txnHandler)); txnHandler.compact(new CompactionRequest("default", tblName, CompactionType.MAJOR)); @@ -1054,7 +1054,7 @@ void testInitiatorWithMultipleFailedCompactionsForVariousTblProperties(String tb numAttemptedCompactions++; checkCompactionState(new CompactionsByState(numAttemptedCompactions,numFailedCompactions + 2,0,0,0,0,numFailedCompactions + 2 + numAttemptedCompactions), countCompacts(txnHandler)); - runHouseKeeperService(compactionHistoryService, hiveConf);//should remove history so that we have + compactionHistoryService.run(); //COMPACTOR_HISTORY_RETENTION_FAILED failed compacts left (and no other since we only have failed ones here) checkCompactionState(new CompactionsByState( hiveConf.getIntVar(HiveConf.ConfVars.COMPACTOR_HISTORY_RETENTION_ATTEMPTED), @@ -1078,7 +1078,7 @@ void testInitiatorWithMultipleFailedCompactionsForVariousTblProperties(String tb hiveConf.getIntVar(HiveConf.ConfVars.COMPACTOR_HISTORY_RETENTION_ATTEMPTED)+ 1), countCompacts(txnHandler)); runCleaner(hiveConf); // transition to Success state - runHouseKeeperService(compactionHistoryService, hiveConf);//should not purge anything as all items within retention sizes + compactionHistoryService.run(); checkCompactionState(new CompactionsByState( hiveConf.getIntVar(HiveConf.ConfVars.COMPACTOR_HISTORY_RETENTION_ATTEMPTED), hiveConf.getIntVar(HiveConf.ConfVars.COMPACTOR_HISTORY_RETENTION_FAILED),0,0,1,0, @@ -1198,26 +1198,6 @@ public static void runCleaner(HiveConf hiveConf) throws MetaException { t.run(); } - public static void runHouseKeeperService(HouseKeeperService houseKeeperService, HiveConf conf) throws Exception { - int lastCount = houseKeeperService.getIsAliveCounter(); - houseKeeperService.start(conf); - int maxIter = 10; - int iterCount = 0; - while(houseKeeperService.getIsAliveCounter() <= lastCount) { - if(iterCount++ >= maxIter) { - //prevent test hangs - throw new IllegalStateException("HouseKeeper didn't run after " + (iterCount - 1) + " waits"); - } - try { - Thread.sleep(100);//make sure it has run at least once - } - catch(InterruptedException ex) { - //... - } - } - houseKeeperService.stop(); - } - /** * HIVE-12352 has details * @throws Exception diff --git ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java index e46e65be42..406bdea96a 100644 --- ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java +++ ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java @@ -19,12 +19,14 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.ThreadPool; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.GetOpenTxnsInfoResponse; import org.apache.hadoop.hive.metastore.api.ShowLocksResponse; import org.apache.hadoop.hive.metastore.api.ShowLocksResponseElement; import org.apache.hadoop.hive.metastore.api.TxnState; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; +import org.apache.hadoop.hive.metastore.txn.AcidHouseKeeperService; import org.apache.hadoop.hive.metastore.txn.TxnDbUtil; import org.apache.hadoop.hive.metastore.txn.TxnStore; import org.apache.hadoop.hive.metastore.txn.TxnUtils; @@ -38,7 +40,6 @@ import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.plan.HiveOperation; import org.apache.hadoop.hive.ql.session.SessionState; -import org.apache.hadoop.hive.ql.txn.AcidHouseKeeperService; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -202,24 +203,9 @@ public void testWriteDynamicPartition() throws Exception { * aborts timed out transactions */ private void runReaper() throws Exception { - int lastCount = houseKeeperService.getIsAliveCounter(); - houseKeeperService.start(conf); - int maxIter = 10; - int iterCount = 0; - while(houseKeeperService.getIsAliveCounter() <= lastCount) { - if(iterCount++ >= maxIter) { - //prevent test hangs - throw new IllegalStateException("Reaper didn't run after " + iterCount + " waits"); - } - try { - Thread.sleep(100);//make sure it has run at least once - } - catch(InterruptedException ex) { - //... - } - } - houseKeeperService.stop(); + houseKeeperService.run(); } + @Test public void testExceptions() throws Exception { addPartitionOutput(newTable(true), WriteEntity.WriteType.INSERT); @@ -491,13 +477,14 @@ public void setUp() throws Exception { conf.setTimeVar(HiveConf.ConfVars.HIVE_TIMEDOUT_TXN_REAPER_START, 0, TimeUnit.SECONDS); conf.setTimeVar(HiveConf.ConfVars.HIVE_TXN_TIMEOUT, 10, TimeUnit.SECONDS); houseKeeperService = new AcidHouseKeeperService(); + houseKeeperService.setConf(conf); } @After public void tearDown() throws Exception { - if(houseKeeperService != null) houseKeeperService.stop(); if (txnMgr != null) txnMgr.closeTxnManager(); TxnDbUtil.cleanDb(conf); + ThreadPool.shutdown(); } private static class MockQueryPlan extends QueryPlan { diff --git ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java index e9833cb5e9..e46ef513e5 100644 --- ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java +++ ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java @@ -19,13 +19,13 @@ import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hive.common.JavaUtils; -import org.apache.hadoop.hive.common.ValidTxnList; +import org.apache.hadoop.hive.metastore.RunnableConfigurable; import org.apache.hadoop.hive.metastore.api.AddDynamicPartitions; import org.apache.hadoop.hive.metastore.api.DataOperationType; +import org.apache.hadoop.hive.metastore.txn.AcidWriteSetService; import org.apache.hadoop.hive.metastore.txn.TxnStore; import org.apache.hadoop.hive.metastore.txn.TxnUtils; import org.apache.hadoop.hive.ql.TestTxnCommands2; -import org.apache.hadoop.hive.ql.txn.AcidWriteSetService; import org.junit.After; import org.junit.Assert; import org.apache.hadoop.hive.common.FileUtils; @@ -42,8 +42,6 @@ import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; import org.apache.hadoop.hive.ql.session.SessionState; import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Ignore; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -54,11 +52,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.Executor; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.ThreadFactory; /** * See additional tests in {@link org.apache.hadoop.hive.ql.lockmgr.TestDbTxnManager} @@ -1013,7 +1006,8 @@ public void testWriteSetTracking4() throws Exception { 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET")); AcidWriteSetService houseKeeper = new AcidWriteSetService(); - TestTxnCommands2.runHouseKeeperService(houseKeeper, conf); + houseKeeper.setConf(conf); + houseKeeper.run(); //since T3 overlaps with Long Running (still open) GC does nothing Assert.assertEquals(1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET")); checkCmdOnDriver(driver.compileAndRespond("update TAB2 set b = 17 where a = 1"));//no rows match @@ -1027,7 +1021,7 @@ public void testWriteSetTracking4() throws Exception { locks = getLocks(txnMgr); Assert.assertEquals("Unexpected lock count", 0, locks.size()); - TestTxnCommands2.runHouseKeeperService(houseKeeper, conf); + houseKeeper.run(); Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET")); } /** @@ -1097,7 +1091,9 @@ public void testWriteSetTracking6() throws Exception { Assert.assertEquals("Unexpected lock count", 1, locks.size()); checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "TAB2", null, locks); txnMgr.commitTxn(); - TestTxnCommands2.runHouseKeeperService(new AcidWriteSetService(), conf); + RunnableConfigurable writeSetService = new AcidWriteSetService(); + writeSetService.setConf(conf); + writeSetService.run(); Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET")); } diff --git shims/common/src/main/java/org/apache/hadoop/hive/io/HdfsUtils.java shims/common/src/main/java/org/apache/hadoop/hive/io/HdfsUtils.java index 277738fac7..85ee8c75b2 100644 --- shims/common/src/main/java/org/apache/hadoop/hive/io/HdfsUtils.java +++ shims/common/src/main/java/org/apache/hadoop/hive/io/HdfsUtils.java @@ -51,17 +51,6 @@ public class HdfsUtils { private static final Logger LOG = LoggerFactory.getLogger("shims.HdfsUtils"); - // TODO: this relies on HDFS not changing the format; we assume if we could get inode ID, this - // is still going to work. Otherwise, file IDs can be turned off. Later, we should use - // as public utility method in HDFS to obtain the inode-based path. - private static final String HDFS_ID_PATH_PREFIX = "/.reserved/.inodes/"; - - public static Path getFileIdPath( - FileSystem fileSystem, Path path, long fileId) { - return (fileSystem instanceof DistributedFileSystem) - ? new Path(HDFS_ID_PATH_PREFIX + fileId) : path; - } - /** * Copy the permissions, group, and ACLs from a source {@link HadoopFileStatus} to a target {@link Path}. This method * will only log a warning if permissions cannot be set, no exception will be thrown. diff --git standalone-metastore/pom.xml standalone-metastore/pom.xml index 07b767bc13..4fea01d831 100644 --- standalone-metastore/pom.xml +++ standalone-metastore/pom.xml @@ -176,6 +176,12 @@ javax.jdo ${datanucleus-jdo.version} + + org.skyscreamer + jsonassert + 1.4.0 + test + diff --git standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ClientCapability.java standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ClientCapability.java index 8fc83110d1..8b0d71c99d 100644 --- standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ClientCapability.java +++ standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ClientCapability.java @@ -1,4 +1,4 @@ -/** +/* * Autogenerated by Thrift Compiler (0.9.3) * * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING @@ -7,17 +7,13 @@ package org.apache.hadoop.hive.metastore.api; -import java.util.Map; -import java.util.HashMap; -import org.apache.thrift.TEnum; - public enum ClientCapability implements org.apache.thrift.TEnum { TEST_CAPABILITY(1), INSERT_ONLY_TABLES(2); private final int value; - private ClientCapability(int value) { + ClientCapability(int value) { this.value = value; } diff --git metastore/src/java/org/apache/hadoop/hive/metastore/AcidEventListener.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/AcidEventListener.java similarity index 81% rename from metastore/src/java/org/apache/hadoop/hive/metastore/AcidEventListener.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/AcidEventListener.java index 71ad916cb3..1512ffbfe1 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/AcidEventListener.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/AcidEventListener.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -19,9 +19,10 @@ package org.apache.hadoop.hive.metastore; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.HiveObjectType; import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent; import org.apache.hadoop.hive.metastore.events.DropPartitionEvent; import org.apache.hadoop.hive.metastore.events.DropTableEvent; @@ -35,11 +36,11 @@ public class AcidEventListener extends MetaStoreEventListener { private TxnStore txnHandler; - private HiveConf hiveConf; + private Configuration conf; public AcidEventListener(Configuration configuration) { super(configuration); - hiveConf = (HiveConf) configuration; + conf = configuration; } @Override @@ -69,24 +70,24 @@ public void onDropPartition(DropPartitionEvent partitionEvent) throws MetaExcep } private TxnStore getTxnHandler() { - boolean hackOn = HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_IN_TEST) || - HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_IN_TEZ_TEST); + boolean hackOn = MetastoreConf.getBoolVar(conf, ConfVars.HIVE_IN_TEST) || + MetastoreConf.getBoolVar(conf, ConfVars.HIVE_IN_TEZ_TEST); String origTxnMgr = null; boolean origConcurrency = false; // Since TxnUtils.getTxnStore calls TxnHandler.setConf -> checkQFileTestHack -> TxnDbUtil.setConfValues, // which may change the values of below two entries, we need to avoid pulluting the original values if (hackOn) { - origTxnMgr = hiveConf.getVar(HiveConf.ConfVars.HIVE_TXN_MANAGER); - origConcurrency = hiveConf.getBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY); + origTxnMgr = MetastoreConf.getVar(conf, ConfVars.HIVE_TXN_MANAGER); + origConcurrency = MetastoreConf.getBoolVar(conf, ConfVars.HIVE_SUPPORT_CONCURRENCY); } - txnHandler = TxnUtils.getTxnStore(hiveConf); + txnHandler = TxnUtils.getTxnStore(conf); // Set them back if (hackOn) { - hiveConf.setVar(HiveConf.ConfVars.HIVE_TXN_MANAGER, origTxnMgr); - hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, origConcurrency); + MetastoreConf.setVar(conf, ConfVars.HIVE_TXN_MANAGER, origTxnMgr); + MetastoreConf.setBoolVar(conf, ConfVars.HIVE_SUPPORT_CONCURRENCY, origConcurrency); } return txnHandler; diff --git metastore/src/java/org/apache/hadoop/hive/metastore/AlterHandler.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/AlterHandler.java similarity index 94% rename from metastore/src/java/org/apache/hadoop/hive/metastore/AlterHandler.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/AlterHandler.java index a3d322f345..fc0b4d7d08 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/AlterHandler.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/AlterHandler.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -27,7 +27,6 @@ import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; /** * Interface for Alter Table and Alter Partition code @@ -36,7 +35,7 @@ /** * @deprecated As of release 2.2.0. Replaced by {@link #alterTable(RawStore, Warehouse, String, - * String, Table, EnvironmentContext, HMSHandler)} + * String, Table, EnvironmentContext, IHMSHandler)} * * handles alter table, the changes could be cascaded to partitions if applicable * @@ -84,11 +83,11 @@ void alterTable(RawStore msdb, Warehouse wh, String dbname, */ void alterTable(RawStore msdb, Warehouse wh, String dbname, String name, Table newTable, EnvironmentContext envContext, - HMSHandler handler) throws InvalidOperationException, MetaException; + IHMSHandler handler) throws InvalidOperationException, MetaException; /** * @deprecated As of release 2.2.0. Replaced by {@link #alterPartition(RawStore, Warehouse, String, - * String, List, Partition, EnvironmentContext, HMSHandler)} + * String, List, Partition, EnvironmentContext, IHMSHandler)} * * handles alter partition * @@ -139,12 +138,12 @@ Partition alterPartition(final RawStore msdb, Warehouse wh, final String dbname, */ Partition alterPartition(final RawStore msdb, Warehouse wh, final String dbname, final String name, final List part_vals, final Partition new_part, EnvironmentContext environmentContext, - HMSHandler handler) + IHMSHandler handler) throws InvalidOperationException, InvalidObjectException, AlreadyExistsException, MetaException; /** * @deprecated As of release 2.2.0. Replaced by {@link #alterPartitions(RawStore, Warehouse, String, - * String, List, EnvironmentContext, HMSHandler)} + * String, List, EnvironmentContext, IHMSHandler)} * * handles alter partitions * @@ -191,6 +190,6 @@ Partition alterPartition(final RawStore msdb, Warehouse wh, final String dbname, */ List alterPartitions(final RawStore msdb, Warehouse wh, final String dbname, final String name, final List new_parts, - EnvironmentContext environmentContext,HMSHandler handler) + EnvironmentContext environmentContext,IHMSHandler handler) throws InvalidOperationException, InvalidObjectException, AlreadyExistsException, MetaException; } \ No newline at end of file diff --git metastore/src/java/org/apache/hadoop/hive/metastore/FileMetadataManager.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/FileMetadataManager.java similarity index 81% rename from metastore/src/java/org/apache/hadoop/hive/metastore/FileMetadataManager.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/FileMetadataManager.java index 67aa1440de..0b8b3102e1 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/FileMetadataManager.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/FileMetadataManager.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,18 +18,14 @@ package org.apache.hadoop.hive.metastore; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.RemoteIterator; -import java.util.LinkedList; - -import java.util.Queue; - import java.util.ArrayList; - import java.util.List; import com.google.common.collect.Lists; @@ -43,24 +39,19 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.hadoop.hive.io.HdfsUtils; -import org.apache.hadoop.hive.metastore.HiveMetaStore.ThreadLocalRawStore; import org.apache.hadoop.hive.metastore.api.FileMetadataExprType; import org.apache.hadoop.hive.metastore.api.MetaException; -import org.apache.hadoop.hive.shims.HadoopShims; -import org.apache.hadoop.hive.shims.ShimLoader; import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.utils.HdfsUtils; public class FileMetadataManager { private static final Log LOG = LogFactory.getLog(FileMetadataManager.class); - private static final HadoopShims SHIMS = ShimLoader.getHadoopShims(); - private final ThreadLocalRawStore tlms; + private final RawStore tlms; private final ExecutorService threadPool; - private final HiveConf conf; + private final Configuration conf; private final class CacheUpdateRequest implements Callable { FileMetadataExprType type; @@ -85,10 +76,10 @@ public Void call() throws Exception { } } - public FileMetadataManager(ThreadLocalRawStore tlms, HiveConf conf) { + public FileMetadataManager(RawStore tlms, Configuration conf) { this.tlms = tlms; this.conf = conf; - int numThreads = HiveConf.getIntVar(conf, ConfVars.METASTORE_HBASE_FILE_METADATA_THREADS); + int numThreads = MetastoreConf.getIntVar(conf, MetastoreConf.ConfVars.FILE_METADATA_THREADS); this.threadPool = Executors.newFixedThreadPool(numThreads, new ThreadFactoryBuilder().setNameFormat("File-Metadata-%d").setDaemon(true).build()); } @@ -119,8 +110,7 @@ private void cacheMetadata(FileMetadataExprType type, String location) // TODO: use the other HdfsUtils here if (!(fs instanceof DistributedFileSystem)) return; try { - fileId = SHIMS.getFileId((DistributedFileSystem)fs, - Path.getPathWithoutSchemeAndAuthority(file).toString()); + fileId = HdfsUtils.getFileId(fs, Path.getPathWithoutSchemeAndAuthority(file).toString()); } catch (UnsupportedOperationException ex) { LOG.error("Cannot cache file metadata for " + location + "; " + fs.getClass().getCanonicalName() + " does not support fileId"); @@ -128,7 +118,7 @@ private void cacheMetadata(FileMetadataExprType type, String location) } LOG.info("Caching file metadata for " + file + " (file ID " + fileId + ")"); file = HdfsUtils.getFileIdPath(fs, file, fileId); - tlms.getMS().getFileMetadataHandler(type).cacheFileMetadata(fileId, fs, file); + tlms.getFileMetadataHandler(type).cacheFileMetadata(fileId, fs, file); } } } diff --git metastore/src/java/org/apache/hadoop/hive/metastore/HMSMetricsListener.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HMSMetricsListener.java similarity index 87% rename from metastore/src/java/org/apache/hadoop/hive/metastore/HMSMetricsListener.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HMSMetricsListener.java index a2ad06f31e..1ee6d97b9a 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/HMSMetricsListener.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HMSMetricsListener.java @@ -54,37 +54,37 @@ public HMSMetricsListener(Configuration config) { @Override public void onCreateDatabase(CreateDatabaseEvent dbEvent) throws MetaException { - HiveMetaStore.HMSHandler.databaseCount.incrementAndGet(); + Metrics.getOrCreateGauge(MetricsConstants.TOTAL_DATABASES).incrementAndGet(); createdDatabases.inc(); } @Override public void onDropDatabase(DropDatabaseEvent dbEvent) throws MetaException { - HiveMetaStore.HMSHandler.databaseCount.decrementAndGet(); + Metrics.getOrCreateGauge(MetricsConstants.TOTAL_DATABASES).decrementAndGet(); deletedDatabases.inc(); } @Override public void onCreateTable(CreateTableEvent tableEvent) throws MetaException { - HiveMetaStore.HMSHandler.tableCount.incrementAndGet(); + Metrics.getOrCreateGauge(MetricsConstants.TOTAL_TABLES).incrementAndGet(); createdTables.inc(); } @Override public void onDropTable(DropTableEvent tableEvent) throws MetaException { - HiveMetaStore.HMSHandler.tableCount.decrementAndGet(); + Metrics.getOrCreateGauge(MetricsConstants.TOTAL_TABLES).decrementAndGet(); deletedTables.inc(); } @Override public void onDropPartition(DropPartitionEvent partitionEvent) throws MetaException { - HiveMetaStore.HMSHandler.partCount.decrementAndGet(); + Metrics.getOrCreateGauge(MetricsConstants.TOTAL_PARTITIONS).decrementAndGet(); deletedParts.inc(); } @Override public void onAddPartition(AddPartitionEvent partitionEvent) throws MetaException { - HiveMetaStore.HMSHandler.partCount.incrementAndGet(); + Metrics.getOrCreateGauge(MetricsConstants.TOTAL_PARTITIONS).incrementAndGet(); createdParts.inc(); } } diff --git metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java similarity index 95% rename from metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java index 65551ad169..c800d4f8e0 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -21,20 +21,21 @@ import com.google.common.collect.Lists; import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.events.AddPartitionEvent; import org.apache.hadoop.hive.metastore.events.AlterPartitionEvent; import org.apache.hadoop.hive.metastore.events.AlterTableEvent; import org.apache.hadoop.hive.metastore.events.CreateTableEvent; import org.apache.hadoop.hive.metastore.events.DropTableEvent; import org.apache.hadoop.hive.metastore.messaging.EventMessage; +import org.apache.hadoop.hive.metastore.utils.FileUtils; +import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.common.StatsSetupConst; -import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; import org.apache.hadoop.hive.metastore.api.ColumnStatistics; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc; @@ -49,10 +50,7 @@ import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; -import org.apache.hadoop.ipc.RemoteException; -import org.apache.hive.common.util.HiveStringUtils; import java.io.IOException; import java.net.URI; @@ -93,7 +91,7 @@ public void alterTable(RawStore msdb, Warehouse wh, String dbname, @Override public void alterTable(RawStore msdb, Warehouse wh, String dbname, String name, Table newt, EnvironmentContext environmentContext, - HMSHandler handler) throws InvalidOperationException, MetaException { + IHMSHandler handler) throws InvalidOperationException, MetaException { name = name.toLowerCase(); dbname = dbname.toLowerCase(); @@ -117,13 +115,13 @@ public void alterTable(RawStore msdb, Warehouse wh, String dbname, } Path srcPath = null; - FileSystem srcFs = null; + FileSystem srcFs; Path destPath = null; FileSystem destFs = null; boolean success = false; boolean dataWasMoved = false; - Table oldt = null; + Table oldt; List transactionalListeners = null; if (handler != null) { transactionalListeners = handler.getTransactionalListeners(); @@ -132,7 +130,7 @@ public void alterTable(RawStore msdb, Warehouse wh, String dbname, try { boolean rename = false; boolean isPartitionedTable = false; - List parts = null; + List parts; // check if table with the new name already exists if (!newTblName.equals(name) || !newDbName.equals(dbname)) { @@ -154,9 +152,8 @@ public void alterTable(RawStore msdb, Warehouse wh, String dbname, isPartitionedTable = true; } - if (HiveConf.getBoolVar(hiveConf, - HiveConf.ConfVars.METASTORE_DISALLOW_INCOMPATIBLE_COL_TYPE_CHANGES, - false)) { + if (MetastoreConf.getBoolVar(hiveConf, + MetastoreConf.ConfVars.DISALLOW_INCOMPATIBLE_COL_TYPE_CHANGES)) { // Throws InvalidOperationException if the new column types are not // compatible with the current column types. checkColTypeChangeCompatible(oldt.getSd().getCols(), newt.getSd().getCols()); @@ -239,7 +236,7 @@ public void alterTable(RawStore msdb, Warehouse wh, String dbname, // also the location field in partition parts = msdb.getPartitions(dbname, name, -1); - Map columnStatsNeedUpdated = new HashMap(); + Map columnStatsNeedUpdated = new HashMap<>(); for (Partition part : parts) { String oldPartLoc = part.getSd().getLocation(); if (dataWasMoved && oldPartLoc.contains(oldTblLocPath)) { @@ -393,10 +390,10 @@ public Partition alterPartition(final RawStore msdb, Warehouse wh, final String @Override public Partition alterPartition(final RawStore msdb, Warehouse wh, final String dbname, final String name, final List part_vals, final Partition new_part, - EnvironmentContext environmentContext, HMSHandler handler) + EnvironmentContext environmentContext, IHMSHandler handler) throws InvalidOperationException, InvalidObjectException, AlreadyExistsException, MetaException { boolean success = false; - Partition oldPart = null; + Partition oldPart; List transactionalListeners = null; if (handler != null) { transactionalListeners = handler.getTransactionalListeners(); @@ -459,11 +456,11 @@ public Partition alterPartition(final RawStore msdb, Warehouse wh, final String } //rename partition - String oldPartLoc = null; - String newPartLoc = null; + String oldPartLoc; + String newPartLoc; Path srcPath = null; Path destPath = null; - FileSystem srcFs = null; + FileSystem srcFs; FileSystem destFs = null; boolean dataWasMoved = false; try { @@ -615,10 +612,10 @@ public Partition alterPartition(final RawStore msdb, Warehouse wh, final String @Override public List alterPartitions(final RawStore msdb, Warehouse wh, final String dbname, final String name, final List new_parts, EnvironmentContext environmentContext, - HMSHandler handler) + IHMSHandler handler) throws InvalidOperationException, InvalidObjectException, AlreadyExistsException, MetaException { - List oldParts = new ArrayList(); - List> partValsList = new ArrayList>(); + List oldParts = new ArrayList<>(); + List> partValsList = new ArrayList<>(); List transactionalListeners = null; if (handler != null) { transactionalListeners = handler.getTransactionalListeners(); @@ -733,14 +730,14 @@ private Path constructRenamedPath(Path defaultNewPath, Path currentPath) { void alterTableUpdateTableColumnStats(RawStore msdb, Table oldTable, Table newTable) throws MetaException, InvalidObjectException { String dbName = oldTable.getDbName().toLowerCase(); - String tableName = HiveStringUtils.normalizeIdentifier(oldTable.getTableName()); + String tableName = org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifier(oldTable.getTableName()); String newDbName = newTable.getDbName().toLowerCase(); - String newTableName = HiveStringUtils.normalizeIdentifier(newTable.getTableName()); + String newTableName = org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifier(newTable.getTableName()); try { List oldCols = oldTable.getSd().getCols(); List newCols = newTable.getSd().getCols(); - List newStatsObjs = new ArrayList(); + List newStatsObjs = new ArrayList<>(); ColumnStatistics colStats = null; boolean updateColumnStats = true; @@ -752,7 +749,7 @@ void alterTableUpdateTableColumnStats(RawStore msdb, Table oldTable, Table newTa } if (updateColumnStats) { - List oldColNames = new ArrayList(oldCols.size()); + List oldColNames = new ArrayList<>(oldCols.size()); for (FieldSchema oldCol : oldCols) { oldColNames.add(oldCol.getName()); } @@ -764,7 +761,7 @@ void alterTableUpdateTableColumnStats(RawStore msdb, Table oldTable, Table newTa } else { List statsObjs = colStats.getStatsObj(); if (statsObjs != null) { - List deletedCols = new ArrayList(); + List deletedCols = new ArrayList<>(); for (ColumnStatisticsObj statsObj : statsObjs) { boolean found = false; for (FieldSchema newCol : newCols) { @@ -815,7 +812,7 @@ private ColumnStatistics updateOrGetPartitionColumnStats( ColumnStatistics newPartsColStats = null; try { List newCols = part.getSd() == null ? - new ArrayList() : part.getSd().getCols(); + new ArrayList<>() : part.getSd().getCols(); String oldPartName = Warehouse.makePartName(table.getPartitionKeys(), partVals); String newPartName = Warehouse.makePartName(table.getPartitionKeys(), part.getValues()); boolean rename = !part.getDbName().equals(dbname) || !part.getTableName().equals(tblname) @@ -825,7 +822,7 @@ private ColumnStatistics updateOrGetPartitionColumnStats( if (!rename && MetaStoreUtils.columnsIncludedByNameType(oldCols, newCols)) { return newPartsColStats; } - List oldColNames = new ArrayList(oldCols.size()); + List oldColNames = new ArrayList<>(oldCols.size()); for (FieldSchema oldCol : oldCols) { oldColNames.add(oldCol.getName()); } @@ -834,9 +831,9 @@ private ColumnStatistics updateOrGetPartitionColumnStats( oldPartNames, oldColNames); assert (partsColStats.size() <= 1); for (ColumnStatistics partColStats : partsColStats) { //actually only at most one loop - List newStatsObjs = new ArrayList(); + List newStatsObjs = new ArrayList<>(); List statsObjs = partColStats.getStatsObj(); - List deletedCols = new ArrayList(); + List deletedCols = new ArrayList<>(); for (ColumnStatisticsObj statsObj : statsObjs) { boolean found =false; for (FieldSchema newCol : newCols) { diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IHMSHandler.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IHMSHandler.java index 633b3c7a17..85bdc4d969 100644 --- standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IHMSHandler.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IHMSHandler.java @@ -18,11 +18,70 @@ package org.apache.hadoop.hive.metastore; +import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; +import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore; +import java.util.List; + +/** + * An interface wrapper for HMSHandler. This interface contains methods that need to be + * called by internal classes but that are not part of the thrift interface. + */ +@InterfaceAudience.Private public interface IHMSHandler extends ThriftHiveMetastore.Iface, Configurable { void init() throws MetaException; + + /** + * Get the id of the thread of this handler. + * @return thread id + */ + int getThreadId(); + + /** + * Get a reference to the underlying RawStore. + * @return the RawStore instance. + * @throws MetaException if the creation of a new RawStore object is necessary but fails. + */ + RawStore getMS() throws MetaException; + + /** + * Get a reference to Hive's warehouse object (the class that does all the physical operations). + * @return Warehouse instance. + */ + Warehouse getWh(); + + /** + * Equivalent to get_database, but does not write to audit logs, or fire pre-event listeners. + * Meant to be used for internal hive classes that don't use the thrift interface. + * @param name database name + * @return database object + * @throws NoSuchObjectException If the database does not exist. + * @throws MetaException If another error occurs. + */ + Database get_database_core(final String name) throws NoSuchObjectException, MetaException; + + /** + * Equivalent of get_table, but does not log audits and fire pre-event listener. + * Meant to be used for calls made by other hive classes, that are not using the + * thrift interface. + * @param dbname database name + * @param name table name + * @return Table object + * @throws NoSuchObjectException If the table does not exist. + * @throws MetaException If another error occurs. + */ + Table get_table_core(final String dbname, final String name) throws MetaException, + NoSuchObjectException; + + /** + * Get a list of all transactional listeners. + * @return list of listeners. + */ + List getTransactionalListeners(); } diff --git metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEventListener.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreEventListener.java similarity index 97% rename from metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEventListener.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreEventListener.java index 5a72082e95..fc4f4d7253 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEventListener.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreEventListener.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,10 +18,10 @@ package org.apache.hadoop.hive.metastore; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.events.AddForeignKeyEvent; import org.apache.hadoop.hive.metastore.events.AddIndexEvent; diff --git metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEventListenerConstants.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreEventListenerConstants.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEventListenerConstants.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreEventListenerConstants.java index 79de79d11f..0b3de494ea 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEventListenerConstants.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreEventListenerConstants.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreInitContext.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreInitContext.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreInitContext.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreInitContext.java index 2ce621e1cd..0968f42f31 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreInitContext.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreInitContext.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreInitListener.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreInitListener.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreInitListener.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreInitListener.java index c0f62bc565..12130eacca 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreInitListener.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreInitListener.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreListenerNotifier.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreListenerNotifier.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreListenerNotifier.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreListenerNotifier.java index 331621ed29..f6e25c6c0a 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreListenerNotifier.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreListenerNotifier.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -21,7 +21,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; -import org.apache.hadoop.hive.common.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hive.metastore.api.EnvironmentContext; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.events.AddForeignKeyEvent; @@ -53,7 +53,7 @@ /** * This class is used to notify a list of listeners about specific MetaStore events. */ -@Private +@InterfaceAudience.Private public class MetaStoreListenerNotifier { private interface EventNotifier { diff --git metastore/src/java/org/apache/hadoop/hive/metastore/MetaStorePreEventListener.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStorePreEventListener.java similarity index 93% rename from metastore/src/java/org/apache/hadoop/hive/metastore/MetaStorePreEventListener.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStorePreEventListener.java index bca6a31f54..a757dff806 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/MetaStorePreEventListener.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStorePreEventListener.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,10 +18,10 @@ package org.apache.hadoop.hive.metastore; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; import org.apache.hadoop.hive.metastore.api.InvalidOperationException; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; diff --git metastore/src/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java similarity index 87% rename from metastore/src/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java index affb38f882..559ff34f57 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -26,13 +26,14 @@ import java.util.concurrent.TimeUnit; import org.apache.commons.lang.exception.ExceptionUtils; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; import org.apache.hadoop.hive.metastore.metrics.PerfLogger; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; import org.datanucleus.exceptions.NucleusException; @@ -58,21 +59,21 @@ public Result(Object result, int numRetries) { private final MetaStoreInit.MetaStoreInitData metaStoreInitData = new MetaStoreInit.MetaStoreInitData(); - private final HiveConf origConf; // base configuration + private final Configuration origConf; // base configuration private final Configuration activeConf; // active configuration - private RetryingHMSHandler(HiveConf hiveConf, IHMSHandler baseHandler, boolean local) throws MetaException { - this.origConf = hiveConf; + private RetryingHMSHandler(Configuration origConf, IHMSHandler baseHandler, boolean local) throws MetaException { + this.origConf = origConf; this.baseHandler = baseHandler; if (local) { - baseHandler.setConf(hiveConf); // tests expect configuration changes applied directly to metastore + baseHandler.setConf(origConf); // tests expect configuration changes applied directly to metastore } activeConf = baseHandler.getConf(); // This has to be called before initializing the instance of HMSHandler // Using the hook on startup ensures that the hook always has priority // over settings in *.xml. The thread local conf needs to be used because at this point // it has already been initialized using hiveConf. - MetaStoreInit.updateConnectionURL(hiveConf, getActiveConf(), null, metaStoreInitData); + MetaStoreInit.updateConnectionURL(origConf, getActiveConf(), null, metaStoreInitData); try { //invoking init method of baseHandler this way since it adds the retry logic //in case of transient failures in init method @@ -86,10 +87,10 @@ private RetryingHMSHandler(HiveConf hiveConf, IHMSHandler baseHandler, boolean l } } - public static IHMSHandler getProxy(HiveConf hiveConf, IHMSHandler baseHandler, boolean local) + public static IHMSHandler getProxy(Configuration conf, IHMSHandler baseHandler, boolean local) throws MetaException { - RetryingHMSHandler handler = new RetryingHMSHandler(hiveConf, baseHandler, local); + RetryingHMSHandler handler = new RetryingHMSHandler(conf, baseHandler, local); return (IHMSHandler) Proxy.newProxyInstance( RetryingHMSHandler.class.getClassLoader(), @@ -99,7 +100,7 @@ public static IHMSHandler getProxy(HiveConf hiveConf, IHMSHandler baseHandler, b @Override public Object invoke(final Object proxy, final Method method, final Object[] args) throws Throwable { int retryCount = -1; - int threadId = HiveMetaStore.HMSHandler.get(); + int threadId = baseHandler.getThreadId(); boolean error = true; PerfLogger perfLogger = PerfLogger.getPerfLogger(false); perfLogger.PerfLogBegin(CLASS_NAME, method.getName()); @@ -119,14 +120,12 @@ public Object invoke(final Object proxy, final Method method, final Object[] arg public Result invokeInternal(final Object proxy, final Method method, final Object[] args) throws Throwable { boolean gotNewConnectUrl = false; - boolean reloadConf = HiveConf.getBoolVar(origConf, - HiveConf.ConfVars.HMSHANDLERFORCERELOADCONF); - long retryInterval = HiveConf.getTimeVar(origConf, - HiveConf.ConfVars.HMSHANDLERINTERVAL, TimeUnit.MILLISECONDS); - int retryLimit = HiveConf.getIntVar(origConf, - HiveConf.ConfVars.HMSHANDLERATTEMPTS); - long timeout = HiveConf.getTimeVar(origConf, - HiveConf.ConfVars.METASTORE_CLIENT_SOCKET_TIMEOUT, TimeUnit.MILLISECONDS); + boolean reloadConf = MetastoreConf.getBoolVar(origConf, ConfVars.HMSHANDLERFORCERELOADCONF); + long retryInterval = MetastoreConf.getTimeVar(origConf, + ConfVars.HMSHANDLERINTERVAL, TimeUnit.MILLISECONDS); + int retryLimit = MetastoreConf.getIntVar(origConf, ConfVars.HMSHANDLERATTEMPTS); + long timeout = MetastoreConf.getTimeVar(origConf, + ConfVars.CLIENT_SOCKET_TIMEOUT, TimeUnit.MILLISECONDS); Deadline.registerIfNot(timeout); diff --git metastore/src/java/org/apache/hadoop/hive/metastore/SessionPropertiesListener.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/SessionPropertiesListener.java similarity index 81% rename from metastore/src/java/org/apache/hadoop/hive/metastore/SessionPropertiesListener.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/SessionPropertiesListener.java index ee9667809f..214e353401 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/SessionPropertiesListener.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/SessionPropertiesListener.java @@ -19,8 +19,8 @@ package org.apache.hadoop.hive.metastore; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.events.ConfigChangeEvent; import java.util.concurrent.TimeUnit; @@ -36,9 +36,10 @@ public SessionPropertiesListener(Configuration configuration) { @Override public void onConfigChange(ConfigChangeEvent changeEvent) throws MetaException { - if (changeEvent.getKey().equals(HiveConf.ConfVars.METASTORE_CLIENT_SOCKET_TIMEOUT.varname)) { + if (changeEvent.getKey().equals(MetastoreConf.ConfVars.CLIENT_SOCKET_TIMEOUT.varname) || + changeEvent.getKey().equals(MetastoreConf.ConfVars.CLIENT_SOCKET_TIMEOUT.hiveName)) { // TODO: this only applies to current thread, so it's not useful at all. - Deadline.resetTimeout(HiveConf.toTime(changeEvent.getNewValue(), TimeUnit.SECONDS, + Deadline.resetTimeout(MetastoreConf.convertTimeStr(changeEvent.getNewValue(), TimeUnit.SECONDS, TimeUnit.MILLISECONDS)); } } diff --git metastore/src/java/org/apache/hadoop/hive/metastore/TransactionalMetaStoreEventListener.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalMetaStoreEventListener.java similarity index 96% rename from metastore/src/java/org/apache/hadoop/hive/metastore/TransactionalMetaStoreEventListener.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalMetaStoreEventListener.java index d7f1c64487..b6a67acdf3 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/TransactionalMetaStoreEventListener.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalMetaStoreEventListener.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -30,7 +30,7 @@ /** * Constructor * - * @param config + * @param config configuration object */ public TransactionalMetaStoreEventListener(Configuration config) { super(config); diff --git metastore/src/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java index 29d8da8082..7d2537d624 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -31,6 +31,7 @@ import org.apache.hadoop.hive.metastore.events.PreAlterTableEvent; import org.apache.hadoop.hive.metastore.events.PreCreateTableEvent; import org.apache.hadoop.hive.metastore.events.PreEventContext; +import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java index 5933318e64..049445ed4b 100644 --- standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java @@ -151,6 +151,7 @@ public String toString() { ConfVars.AGGREGATE_STATS_CACHE_MAX_FULL, ConfVars.AGGREGATE_STATS_CACHE_CLEAN_UNTIL, ConfVars.DISALLOW_INCOMPATIBLE_COL_TYPE_CHANGES, + ConfVars.FILE_METADATA_THREADS }; /** @@ -430,6 +431,9 @@ public static ConfVars getMetaConf(String name) { EXPRESSION_PROXY_CLASS("metastore.expression.proxy", "hive.metastore.expression.proxy", "org.apache.hadoop.hive.ql.optimizer.ppr.PartitionExpressionForMetastore", "Class to use to process expressions in partition pruning."), + FILE_METADATA_THREADS("metastore.file.metadata.threads", + "hive.metastore.hbase.file.metadata.threads", 1, + "Number of threads to use to read file metadata in background to cache it."), FILTER_HOOK("metastore.filter.hook", "hive.metastore.filter.hook", "org.apache.hadoop.hive.metastore.DefaultMetaStoreFilterHookImpl", "Metastore hook class for filtering the metadata read results. If hive.security.authorization.manager" @@ -485,6 +489,18 @@ public static ConfVars getMetaConf(String name) { "javax.jdo.PersistenceManagerFactoryClass", "org.datanucleus.api.jdo.JDOPersistenceManagerFactory", "class implementing the jdo persistence"), + // Parameters for exporting metadata on table drop (requires the use of the) + // org.apache.hadoop.hive.ql.parse.MetaDataExportListener preevent listener + METADATA_EXPORT_LOCATION("metastore.metadata.export.location", "hive.metadata.export.location", + "", + "When used in conjunction with the org.apache.hadoop.hive.ql.parse.MetaDataExportListener pre event listener, \n" + + "it is the location to which the metadata will be exported. The default is an empty string, which results in the \n" + + "metadata being exported to the current user's home directory on HDFS."), + MOVE_EXPORTED_METADATA_TO_TRASH("metastore.metadata.move.exported.metadata.to.trash", + "hive.metadata.move.exported.metadata.to.trash", true, + "When used in conjunction with the org.apache.hadoop.hive.ql.parse.MetaDataExportListener pre event listener, \n" + + "this setting determines if the metadata that is exported will subsequently be moved to the user's trash directory \n" + + "alongside the dropped table data. This ensures that the metadata will be cleaned up along with the dropped table data."), METRICS_ENABLED("metastore.metrics.enabled", "hive.metastore.metrics.enabled", false, "Enable metrics on the metastore."), METRICS_JSON_FILE_INTERVAL("metastore.metrics.file.frequency", diff --git metastore/src/java/org/apache/hadoop/hive/metastore/events/AddForeignKeyEvent.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/AddForeignKeyEvent.java similarity index 82% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/AddForeignKeyEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/AddForeignKeyEvent.java index 1dc95889d1..f9c951c59f 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/events/AddForeignKeyEvent.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/AddForeignKeyEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -20,13 +20,17 @@ import java.util.List; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.SQLForeignKey; +@InterfaceStability.Stable +@InterfaceAudience.Public public class AddForeignKeyEvent extends ListenerEvent { private final List fks; - public AddForeignKeyEvent(List fks, boolean status, HMSHandler handler) { + public AddForeignKeyEvent(List fks, boolean status, IHMSHandler handler) { super(status, handler); this.fks = fks; } diff --git metastore/src/java/org/apache/hadoop/hive/metastore/events/AddIndexEvent.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/AddIndexEvent.java similarity index 79% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/AddIndexEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/AddIndexEvent.java index 57fe5d1a15..0d6cb5649c 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/events/AddIndexEvent.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/AddIndexEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,9 +18,9 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.Index; @InterfaceAudience.Public @@ -29,7 +29,7 @@ private final Index index; - public AddIndexEvent(Index index, boolean status, HiveMetaStore.HMSHandler handler) { + public AddIndexEvent(Index index, boolean status, IHMSHandler handler) { super(status, handler); this.index = index; } diff --git metastore/src/java/org/apache/hadoop/hive/metastore/events/AddNotNullConstraintEvent.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/AddNotNullConstraintEvent.java similarity index 80% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/AddNotNullConstraintEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/AddNotNullConstraintEvent.java index c01083cd88..6b4cf0b5ea 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/events/AddNotNullConstraintEvent.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/AddNotNullConstraintEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -20,13 +20,18 @@ import java.util.List; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint; +@InterfaceAudience.Public +@InterfaceStability.Stable public class AddNotNullConstraintEvent extends ListenerEvent { private final List nns; - public AddNotNullConstraintEvent(List nns, boolean status, HMSHandler handler) { + public AddNotNullConstraintEvent(List nns, boolean status, + IHMSHandler handler) { super(status, handler); this.nns = nns; } diff --git metastore/src/java/org/apache/hadoop/hive/metastore/events/AddPartitionEvent.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/AddPartitionEvent.java similarity index 87% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/AddPartitionEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/AddPartitionEvent.java index 85f5fb84de..d4542d7a5b 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/events/AddPartitionEvent.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/AddPartitionEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,9 +18,9 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy; @@ -37,21 +37,23 @@ private final List partitions; private PartitionSpecProxy partitionSpecProxy; - public AddPartitionEvent(Table table, List partitions, boolean status, HMSHandler handler) { + public AddPartitionEvent(Table table, List partitions, boolean status, + IHMSHandler handler) { super(status, handler); this.table = table; this.partitions = partitions; this.partitionSpecProxy = null; } - public AddPartitionEvent(Table table, Partition partition, boolean status, HMSHandler handler) { + public AddPartitionEvent(Table table, Partition partition, boolean status, IHMSHandler handler) { this(table, Arrays.asList(partition), status, handler); } /** * Alternative constructor to use PartitionSpec APIs. */ - public AddPartitionEvent(Table table, PartitionSpecProxy partitionSpec, boolean status, HMSHandler handler) { + public AddPartitionEvent(Table table, PartitionSpecProxy partitionSpec, boolean status, + IHMSHandler handler) { super(status, handler); this.table = table; this.partitions = null; diff --git metastore/src/java/org/apache/hadoop/hive/metastore/events/AddPrimaryKeyEvent.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/AddPrimaryKeyEvent.java similarity index 82% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/AddPrimaryKeyEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/AddPrimaryKeyEvent.java index cb0f562c41..de7978cfbd 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/events/AddPrimaryKeyEvent.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/AddPrimaryKeyEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -20,14 +20,18 @@ import java.util.List; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey; +@InterfaceAudience.Public +@InterfaceStability.Stable public class AddPrimaryKeyEvent extends ListenerEvent { private final List pks; - public AddPrimaryKeyEvent(List pks, boolean status, HMSHandler handler) { + public AddPrimaryKeyEvent(List pks, boolean status, IHMSHandler handler) { super(status, handler); this.pks = pks; } diff --git metastore/src/java/org/apache/hadoop/hive/metastore/events/AddUniqueConstraintEvent.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/AddUniqueConstraintEvent.java similarity index 80% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/AddUniqueConstraintEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/AddUniqueConstraintEvent.java index c02a3099f3..b677ee6427 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/events/AddUniqueConstraintEvent.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/AddUniqueConstraintEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -20,13 +20,18 @@ import java.util.List; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint; +@InterfaceAudience.Public +@InterfaceStability.Stable public class AddUniqueConstraintEvent extends ListenerEvent { private final List uks; - public AddUniqueConstraintEvent(List uks, boolean status, HMSHandler handler) { + public AddUniqueConstraintEvent(List uks, boolean status, + IHMSHandler handler) { super(status, handler); this.uks = uks; } diff --git metastore/src/java/org/apache/hadoop/hive/metastore/events/AlterIndexEvent.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/AlterIndexEvent.java similarity index 84% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/AlterIndexEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/AlterIndexEvent.java index 99e51f6039..2c72b83635 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/events/AlterIndexEvent.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/AlterIndexEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,9 +18,9 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.Index; @InterfaceAudience.Public @@ -30,8 +30,7 @@ private final Index newIndex; private final Index oldIndex; - public AlterIndexEvent(Index oldIndex, Index newIndex, boolean status, - HiveMetaStore.HMSHandler handler) { + public AlterIndexEvent(Index oldIndex, Index newIndex, boolean status, IHMSHandler handler) { super(status, handler); this.oldIndex = oldIndex; this.newIndex = newIndex; diff --git metastore/src/java/org/apache/hadoop/hive/metastore/events/AlterPartitionEvent.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/AlterPartitionEvent.java similarity index 87% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/AlterPartitionEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/AlterPartitionEvent.java index 4025f9f504..09cad85334 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/events/AlterPartitionEvent.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/AlterPartitionEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,9 +18,9 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.Table; @@ -34,7 +34,7 @@ private final boolean isTruncateOp; public AlterPartitionEvent(Partition oldPart, Partition newPart, Table table, boolean isTruncateOp, - boolean status, HMSHandler handler) { + boolean status, IHMSHandler handler) { super(status, handler); this.oldPart = oldPart; this.newPart = newPart; diff --git metastore/src/java/org/apache/hadoop/hive/metastore/events/AlterTableEvent.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/AlterTableEvent.java similarity index 85% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/AlterTableEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/AlterTableEvent.java index eb9d94fcc3..2e3f6f00ee 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/events/AlterTableEvent.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/AlterTableEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file @@ -19,9 +19,9 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.Table; @InterfaceAudience.Public @@ -32,7 +32,8 @@ private final Table oldTable; private final boolean isTruncateOp; - public AlterTableEvent (Table oldTable, Table newTable, boolean isTruncateOp, boolean status, HMSHandler handler) { + public AlterTableEvent (Table oldTable, Table newTable, boolean isTruncateOp, boolean status, + IHMSHandler handler) { super (status, handler); this.oldTable = oldTable; this.newTable = newTable; diff --git metastore/src/java/org/apache/hadoop/hive/metastore/events/ConfigChangeEvent.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/ConfigChangeEvent.java similarity index 80% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/ConfigChangeEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/ConfigChangeEvent.java index 9d23dc8c13..090d175f1f 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/events/ConfigChangeEvent.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/ConfigChangeEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,9 +18,9 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; @InterfaceAudience.Public @InterfaceStability.Stable @@ -30,8 +30,8 @@ private final String oldValue; private final String newValue; - public ConfigChangeEvent(HiveMetaStore.HMSHandler handler, String key, - String oldValue, String newValue) { + public ConfigChangeEvent(IHMSHandler handler, String key, + String oldValue, String newValue) { super(true, handler); this.key = key; this.oldValue = oldValue; diff --git metastore/src/java/org/apache/hadoop/hive/metastore/events/CreateDatabaseEvent.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/CreateDatabaseEvent.java similarity index 80% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/CreateDatabaseEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/CreateDatabaseEvent.java index e853aec138..e2c3ee3085 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/events/CreateDatabaseEvent.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/CreateDatabaseEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,9 +18,9 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.Database; @InterfaceAudience.Public @@ -29,7 +29,7 @@ private final Database db; - public CreateDatabaseEvent (Database db, boolean status, HMSHandler handler) { + public CreateDatabaseEvent (Database db, boolean status, IHMSHandler handler) { super (status, handler); this.db = db; } diff --git metastore/src/java/org/apache/hadoop/hive/metastore/events/CreateFunctionEvent.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/CreateFunctionEvent.java similarity index 84% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/CreateFunctionEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/CreateFunctionEvent.java index 0c6ab17d6d..c1403ec09b 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/events/CreateFunctionEvent.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/CreateFunctionEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,9 +18,9 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.Function; @InterfaceAudience.Public @@ -29,7 +29,7 @@ private final Function function; - public CreateFunctionEvent (Function function, boolean status, HMSHandler handler) { + public CreateFunctionEvent (Function function, boolean status, IHMSHandler handler) { super (status, handler); this.function = function; } diff --git metastore/src/java/org/apache/hadoop/hive/metastore/events/CreateTableEvent.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/CreateTableEvent.java similarity index 80% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/CreateTableEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/CreateTableEvent.java index 09a40bb98e..4f5e887ac2 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/events/CreateTableEvent.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/CreateTableEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,9 +18,9 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.Table; @InterfaceAudience.Public @@ -29,7 +29,7 @@ private final Table table; - public CreateTableEvent (Table table, boolean status, HMSHandler handler) { + public CreateTableEvent (Table table, boolean status, IHMSHandler handler) { super (status, handler); this.table = table; } diff --git metastore/src/java/org/apache/hadoop/hive/metastore/events/DropConstraintEvent.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/DropConstraintEvent.java similarity index 83% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/DropConstraintEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/DropConstraintEvent.java index 53968633b8..4c5918f1c3 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/events/DropConstraintEvent.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/DropConstraintEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,15 +18,19 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; +@InterfaceAudience.Public +@InterfaceStability.Stable public class DropConstraintEvent extends ListenerEvent { private final String dbName; private final String tableName; private final String constraintName; public DropConstraintEvent(String dbName, String tableName, String constraintName, - boolean status, HMSHandler handler) { + boolean status, IHMSHandler handler) { super(status, handler); this.dbName = dbName; this.tableName = tableName; diff --git metastore/src/java/org/apache/hadoop/hive/metastore/events/DropDatabaseEvent.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/DropDatabaseEvent.java similarity index 80% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/DropDatabaseEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/DropDatabaseEvent.java index 92e95a6f21..94fe264206 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/events/DropDatabaseEvent.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/DropDatabaseEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,9 +18,9 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.Database; @InterfaceAudience.Public @@ -29,7 +29,7 @@ private final Database db; - public DropDatabaseEvent(Database db, boolean status, HMSHandler handler) { + public DropDatabaseEvent(Database db, boolean status, IHMSHandler handler) { super (status, handler); this.db = db; } diff --git metastore/src/java/org/apache/hadoop/hive/metastore/events/DropFunctionEvent.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/DropFunctionEvent.java similarity index 80% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/DropFunctionEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/DropFunctionEvent.java index 52f04ca46e..e8728a5904 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/events/DropFunctionEvent.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/DropFunctionEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,9 +18,9 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.Function; @InterfaceAudience.Public @@ -29,7 +29,7 @@ private final Function function; - public DropFunctionEvent(Function function, boolean status, HMSHandler handler) { + public DropFunctionEvent(Function function, boolean status, IHMSHandler handler) { super(status, handler); this.function = function; } diff --git metastore/src/java/org/apache/hadoop/hive/metastore/events/DropIndexEvent.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/DropIndexEvent.java similarity index 79% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/DropIndexEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/DropIndexEvent.java index 9b75055353..4d6983469b 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/events/DropIndexEvent.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/DropIndexEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,9 +18,9 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.Index; @InterfaceAudience.Public @@ -29,7 +29,7 @@ private final Index index; - public DropIndexEvent(Index index, boolean status, HiveMetaStore.HMSHandler handler) { + public DropIndexEvent(Index index, boolean status, IHMSHandler handler) { super(status, handler); this.index = index; } diff --git metastore/src/java/org/apache/hadoop/hive/metastore/events/DropPartitionEvent.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/DropPartitionEvent.java similarity index 86% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/DropPartitionEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/DropPartitionEvent.java index d8f2ea968c..dce9df3a11 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/events/DropPartitionEvent.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/DropPartitionEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,9 +18,9 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.Table; @@ -36,7 +36,7 @@ private final boolean deleteData; public DropPartitionEvent (Table table, - Partition partition, boolean status, boolean deleteData, HMSHandler handler) { + Partition partition, boolean status, boolean deleteData, IHMSHandler handler) { super (status, handler); this.table = table; this.partitions = Collections.singletonList(partition); diff --git metastore/src/java/org/apache/hadoop/hive/metastore/events/DropTableEvent.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/DropTableEvent.java similarity index 86% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/DropTableEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/DropTableEvent.java index 2f2ea9dc01..9152232aae 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/events/DropTableEvent.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/DropTableEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,9 +18,9 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.Table; @InterfaceAudience.Public @@ -30,7 +30,7 @@ private final Table table; private final boolean deleteData; - public DropTableEvent(Table table, boolean status, boolean deleteData, HMSHandler handler) { + public DropTableEvent(Table table, boolean status, boolean deleteData, IHMSHandler handler) { super(status, handler); this.table = table; // In HiveMetaStore, the deleteData flag indicates whether DFS data should be diff --git metastore/src/java/org/apache/hadoop/hive/metastore/events/EventCleanerTask.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/EventCleanerTask.java similarity index 90% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/EventCleanerTask.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/EventCleanerTask.java index 7f99f18828..230c0d3b16 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/events/EventCleanerTask.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/EventCleanerTask.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -20,17 +20,17 @@ import java.util.TimerTask; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; import org.apache.hadoop.hive.metastore.RawStore; public class EventCleanerTask extends TimerTask{ public static final Logger LOG = LoggerFactory.getLogger(EventCleanerTask.class); - private final HMSHandler handler; + private final IHMSHandler handler; - public EventCleanerTask(HMSHandler handler) { + public EventCleanerTask(IHMSHandler handler) { super(); this.handler = handler; } diff --git metastore/src/java/org/apache/hadoop/hive/metastore/events/InsertEvent.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/InsertEvent.java similarity index 70% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/InsertEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/InsertEvent.java index d451122a59..b963f78c06 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/events/InsertEvent.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/InsertEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,16 +18,20 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; +import com.google.common.collect.Lists; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; +import org.apache.hadoop.hive.metastore.api.ClientCapabilities; +import org.apache.hadoop.hive.metastore.api.ClientCapability; import org.apache.hadoop.hive.metastore.api.GetTableRequest; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; import org.apache.hadoop.hive.metastore.api.InsertEventRequestData; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; +import org.apache.thrift.TException; import java.util.ArrayList; import java.util.List; @@ -40,7 +44,7 @@ private final Partition ptnObj; private final boolean replace; private final List files; - private List fileChecksums = new ArrayList(); + private List fileChecksums = new ArrayList<>(); /** * @@ -52,17 +56,28 @@ * @param handler handler that is firing the event */ public InsertEvent(String db, String table, List partVals, - InsertEventRequestData insertData, boolean status, HMSHandler handler) throws MetaException, + InsertEventRequestData insertData, boolean status, IHMSHandler handler) throws MetaException, NoSuchObjectException { super(status, handler); GetTableRequest req = new GetTableRequest(db, table); - req.setCapabilities(HiveMetaStoreClient.TEST_VERSION); - this.tableObj = handler.get_table_req(req).getTable(); - if (partVals != null) { - this.ptnObj = handler.get_partition(db, table, partVals); - } else { - this.ptnObj = null; + // TODO MS-SPLIT Switch this back once HiveMetaStoreClient is moved. + //req.setCapabilities(HiveMetaStoreClient.TEST_VERSION); + req.setCapabilities(new ClientCapabilities( + Lists.newArrayList(ClientCapability.TEST_CAPABILITY))); + try { + this.tableObj = handler.get_table_req(req).getTable(); + if (partVals != null) { + this.ptnObj = handler.get_partition(db, table, partVals); + } else { + this.ptnObj = null; + } + } catch (NoSuchObjectException e) { + // This is to mimic previous behavior where NoSuchObjectException was thrown through this + // method. + throw e; + } catch (TException e) { + throw MetaStoreUtils.newMetaException(e); } // If replace flag is not set by caller, then by default set it to true to maintain backward compatibility diff --git metastore/src/java/org/apache/hadoop/hive/metastore/events/ListenerEvent.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/ListenerEvent.java similarity index 94% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/ListenerEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/ListenerEvent.java index cd66f10ba0..f4f46d8ea4 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/events/ListenerEvent.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/ListenerEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,9 +18,9 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.EnvironmentContext; import javax.annotation.concurrent.NotThreadSafe; @@ -42,7 +42,7 @@ * status of the event, whether event was successful or not. */ private final boolean status; - private final HMSHandler handler; + private final IHMSHandler handler; /** * Key/value parameters used by listeners to store notifications results @@ -67,7 +67,7 @@ // Properties passed by the client, to be used in execution hooks. private EnvironmentContext environmentContext = null; - public ListenerEvent(boolean status, HMSHandler handler) { + public ListenerEvent(boolean status, IHMSHandler handler) { super(); this.status = status; this.handler = handler; @@ -102,7 +102,7 @@ public EnvironmentContext getEnvironmentContext() { /** * @return the handler */ - public HMSHandler getHandler() { + public IHMSHandler getHandler() { return handler; } diff --git metastore/src/java/org/apache/hadoop/hive/metastore/events/LoadPartitionDoneEvent.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/LoadPartitionDoneEvent.java similarity index 84% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/LoadPartitionDoneEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/LoadPartitionDoneEvent.java index b56b3bb3d4..3128a8f6a4 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/events/LoadPartitionDoneEvent.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/LoadPartitionDoneEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -20,9 +20,9 @@ import java.util.Map; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.Table; @InterfaceAudience.Public @@ -34,7 +34,7 @@ private final Map partSpec; public LoadPartitionDoneEvent(boolean status, Table table, - Map partSpec, HMSHandler handler) { + Map partSpec, IHMSHandler handler) { super(status, handler); this.table = table; this.partSpec = partSpec; diff --git metastore/src/java/org/apache/hadoop/hive/metastore/events/PreAddIndexEvent.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreAddIndexEvent.java similarity index 80% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/PreAddIndexEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreAddIndexEvent.java index 94a8836e7f..d9a53f85b7 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/events/PreAddIndexEvent.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreAddIndexEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,9 +18,9 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.Index; @InterfaceAudience.Public @@ -29,7 +29,7 @@ private final Index table; - public PreAddIndexEvent(Index table, HiveMetaStore.HMSHandler handler) { + public PreAddIndexEvent(Index table, IHMSHandler handler) { super(PreEventType.ADD_INDEX, handler); this.table = table; } diff --git metastore/src/java/org/apache/hadoop/hive/metastore/events/PreAddPartitionEvent.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreAddPartitionEvent.java similarity index 88% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/PreAddPartitionEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreAddPartitionEvent.java index 99105f3e56..b5c4607d14 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/events/PreAddPartitionEvent.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreAddPartitionEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,9 +18,9 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy; @@ -37,21 +37,21 @@ private final List partitions; private PartitionSpecProxy partitionSpecProxy; - public PreAddPartitionEvent (Table table, List partitions, HMSHandler handler) { + public PreAddPartitionEvent (Table table, List partitions, IHMSHandler handler) { super(PreEventType.ADD_PARTITION, handler); this.table = table; this.partitions = partitions; this.partitionSpecProxy = null; } - public PreAddPartitionEvent(Table table, Partition partition, HMSHandler handler) { + public PreAddPartitionEvent(Table table, Partition partition, IHMSHandler handler) { this(table, Arrays.asList(partition), handler); } /** * Alternative constructor, using */ - public PreAddPartitionEvent(Table table, PartitionSpecProxy partitionSpecProxy, HMSHandler handler) { + public PreAddPartitionEvent(Table table, PartitionSpecProxy partitionSpecProxy, IHMSHandler handler) { this(table, (List)null, handler); this.partitionSpecProxy = partitionSpecProxy; } diff --git metastore/src/java/org/apache/hadoop/hive/metastore/events/PreAlterDatabaseEvent.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreAlterDatabaseEvent.java similarity index 89% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/PreAlterDatabaseEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreAlterDatabaseEvent.java index 62e2674cd3..6ac1a4eff8 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/events/PreAlterDatabaseEvent.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreAlterDatabaseEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,14 +18,14 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.Database; public class PreAlterDatabaseEvent extends PreEventContext { private final Database oldDB, newDB; - public PreAlterDatabaseEvent(Database oldDB, Database newDB, HMSHandler handler) { + public PreAlterDatabaseEvent(Database oldDB, Database newDB, IHMSHandler handler) { super (PreEventType.ALTER_DATABASE, handler); this.oldDB = oldDB; this.newDB = newDB; diff --git metastore/src/java/org/apache/hadoop/hive/metastore/events/PreAlterIndexEvent.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreAlterIndexEvent.java similarity index 81% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/PreAlterIndexEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreAlterIndexEvent.java index a2b7662b2d..2e9d9d77dd 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/events/PreAlterIndexEvent.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreAlterIndexEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,9 +18,9 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.Index; @InterfaceAudience.Public @@ -30,7 +30,7 @@ private final Index newIndex; private final Index oldIndex; - public PreAlterIndexEvent(Index oldIndex, Index newIndex, HiveMetaStore.HMSHandler handler) { + public PreAlterIndexEvent(Index oldIndex, Index newIndex, IHMSHandler handler) { super(PreEventType.ALTER_INDEX, handler); this.oldIndex = oldIndex; this.newIndex = newIndex; diff --git metastore/src/java/org/apache/hadoop/hive/metastore/events/PreAlterPartitionEvent.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreAlterPartitionEvent.java similarity index 87% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/PreAlterPartitionEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreAlterPartitionEvent.java index 5f29699179..a269563e60 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/events/PreAlterPartitionEvent.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreAlterPartitionEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -20,9 +20,9 @@ import java.util.List; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.Partition; @InterfaceAudience.Public @@ -35,7 +35,7 @@ private final Partition newPart; public PreAlterPartitionEvent(String dbName, String tableName, List oldPartVals, - Partition newPart, HMSHandler handler) { + Partition newPart, IHMSHandler handler) { super(PreEventType.ALTER_PARTITION, handler); this.dbName = dbName; this.tableName = tableName; diff --git metastore/src/java/org/apache/hadoop/hive/metastore/events/PreAlterTableEvent.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreAlterTableEvent.java similarity index 82% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/PreAlterTableEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreAlterTableEvent.java index 4e82ad1aec..31db0e9cb4 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/events/PreAlterTableEvent.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreAlterTableEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file @@ -19,9 +19,9 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.Table; @InterfaceAudience.Public @@ -31,7 +31,7 @@ private final Table newTable; private final Table oldTable; - public PreAlterTableEvent (Table oldTable, Table newTable, HMSHandler handler) { + public PreAlterTableEvent (Table oldTable, Table newTable, IHMSHandler handler) { super (PreEventType.ALTER_TABLE, handler); this.oldTable = oldTable; this.newTable = newTable; diff --git metastore/src/java/org/apache/hadoop/hive/metastore/events/PreAuthorizationCallEvent.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreAuthorizationCallEvent.java similarity index 79% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/PreAuthorizationCallEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreAuthorizationCallEvent.java index 54f6dee12a..1205fc935f 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/events/PreAuthorizationCallEvent.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreAuthorizationCallEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,15 +18,15 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; @InterfaceAudience.Public @InterfaceStability.Stable public class PreAuthorizationCallEvent extends PreEventContext { - public PreAuthorizationCallEvent (HMSHandler handler) { + public PreAuthorizationCallEvent (IHMSHandler handler) { super(PreEventType.AUTHORIZATION_API_CALL, handler); } diff --git metastore/src/java/org/apache/hadoop/hive/metastore/events/PreCreateDatabaseEvent.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreCreateDatabaseEvent.java similarity index 81% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/PreCreateDatabaseEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreCreateDatabaseEvent.java index a5f87e11ca..86ff1a077b 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/events/PreCreateDatabaseEvent.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreCreateDatabaseEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,9 +18,9 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.Database; @InterfaceAudience.Public @@ -29,7 +29,7 @@ private final Database db; - public PreCreateDatabaseEvent (Database db, HMSHandler handler) { + public PreCreateDatabaseEvent (Database db, IHMSHandler handler) { super (PreEventType.CREATE_DATABASE, handler); this.db = db; } diff --git metastore/src/java/org/apache/hadoop/hive/metastore/events/PreCreateTableEvent.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreCreateTableEvent.java similarity index 81% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/PreCreateTableEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreCreateTableEvent.java index 799ad97347..bd3aa3d8ce 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/events/PreCreateTableEvent.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreCreateTableEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,9 +18,9 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.Table; @InterfaceAudience.Public @@ -29,7 +29,7 @@ private final Table table; - public PreCreateTableEvent(Table table, HMSHandler handler) { + public PreCreateTableEvent(Table table, IHMSHandler handler) { super(PreEventType.CREATE_TABLE, handler); this.table = table; } diff --git metastore/src/java/org/apache/hadoop/hive/metastore/events/PreDropDatabaseEvent.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreDropDatabaseEvent.java similarity index 81% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/PreDropDatabaseEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreDropDatabaseEvent.java index adcca6f062..f5ebbcf24a 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/events/PreDropDatabaseEvent.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreDropDatabaseEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,9 +18,9 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.Database; @InterfaceAudience.Public @@ -29,7 +29,7 @@ private final Database db; - public PreDropDatabaseEvent(Database db, HMSHandler handler) { + public PreDropDatabaseEvent(Database db, IHMSHandler handler) { super(PreEventType.DROP_DATABASE, handler); this.db = db; } diff --git metastore/src/java/org/apache/hadoop/hive/metastore/events/PreDropIndexEvent.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreDropIndexEvent.java similarity index 80% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/PreDropIndexEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreDropIndexEvent.java index c7fd44b234..ad570db23d 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/events/PreDropIndexEvent.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreDropIndexEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,9 +18,9 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.Index; @InterfaceAudience.Public @@ -29,7 +29,7 @@ private final Index index; - public PreDropIndexEvent(Index index, HiveMetaStore.HMSHandler handler) { + public PreDropIndexEvent(Index index, IHMSHandler handler) { super(PreEventType.DROP_INDEX, handler); this.index = index; } diff --git metastore/src/java/org/apache/hadoop/hive/metastore/events/PreDropPartitionEvent.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreDropPartitionEvent.java similarity index 84% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/PreDropPartitionEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreDropPartitionEvent.java index e890dedc9c..a0daf3177c 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/events/PreDropPartitionEvent.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreDropPartitionEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,9 +18,9 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.Table; @@ -35,8 +35,8 @@ private final Table table; private final boolean deleteData; - public PreDropPartitionEvent (Table table, - Partition partition, boolean deleteData, HMSHandler handler) { + public PreDropPartitionEvent (Table table, Partition partition, boolean deleteData, + IHMSHandler handler) { super (PreEventType.DROP_PARTITION, handler); this.partitions = Collections.singletonList(partition); this.table = table; diff --git metastore/src/java/org/apache/hadoop/hive/metastore/events/PreDropTableEvent.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreDropTableEvent.java similarity index 83% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/PreDropTableEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreDropTableEvent.java index 6c7f169a90..f06fdcd034 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/events/PreDropTableEvent.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreDropTableEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,9 +18,9 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.Table; @InterfaceAudience.Public @@ -30,7 +30,7 @@ private final Table table; private final boolean deleteData; - public PreDropTableEvent(Table table, boolean deleteData, HMSHandler handler) { + public PreDropTableEvent(Table table, boolean deleteData, IHMSHandler handler) { super(PreEventType.DROP_TABLE, handler); this.table = table; // In HiveMetaStore, the deleteData flag indicates whether DFS data should be diff --git metastore/src/java/org/apache/hadoop/hive/metastore/events/PreEventContext.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreEventContext.java similarity index 80% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/PreEventContext.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreEventContext.java index 0bd2ea6400..a3f4836fa9 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/events/PreEventContext.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreEventContext.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,10 +18,9 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; - +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; /** * Base class for all the events which are defined for metastore. @@ -30,7 +29,7 @@ @InterfaceStability.Stable public abstract class PreEventContext { - public static enum PreEventType { + public enum PreEventType { CREATE_TABLE, DROP_TABLE, ALTER_TABLE, @@ -50,9 +49,9 @@ } private final PreEventType eventType; - private final HMSHandler handler; + private final IHMSHandler handler; - public PreEventContext(PreEventType eventType, HMSHandler handler) { + public PreEventContext(PreEventType eventType, IHMSHandler handler) { this.eventType = eventType; this.handler = handler; } @@ -67,7 +66,7 @@ public PreEventType getEventType() { /** * @return the handler */ - public HMSHandler getHandler() { + public IHMSHandler getHandler() { return handler; } diff --git metastore/src/java/org/apache/hadoop/hive/metastore/events/PreLoadPartitionDoneEvent.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreLoadPartitionDoneEvent.java similarity index 85% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/PreLoadPartitionDoneEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreLoadPartitionDoneEvent.java index 82b72eda8a..999ec31f65 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/events/PreLoadPartitionDoneEvent.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreLoadPartitionDoneEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,11 +18,11 @@ package org.apache.hadoop.hive.metastore.events; -import java.util.Map; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import java.util.Map; @InterfaceAudience.Public @InterfaceStability.Stable @@ -33,7 +33,7 @@ private final Map partSpec; public PreLoadPartitionDoneEvent(String dbName, String tableName, - Map partSpec, HMSHandler handler) { + Map partSpec, IHMSHandler handler) { super(PreEventType.LOAD_PARTITION_DONE, handler); this.dbName = dbName; this.tableName = tableName; diff --git metastore/src/java/org/apache/hadoop/hive/metastore/events/PreReadDatabaseEvent.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreReadDatabaseEvent.java similarity index 81% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/PreReadDatabaseEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreReadDatabaseEvent.java index f223960449..88f3855bbc 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/events/PreReadDatabaseEvent.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreReadDatabaseEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,9 +18,9 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.Database; /** @@ -31,7 +31,7 @@ public class PreReadDatabaseEvent extends PreEventContext { private final Database db; - public PreReadDatabaseEvent(Database db, HMSHandler handler) { + public PreReadDatabaseEvent(Database db, IHMSHandler handler) { super(PreEventType.READ_DATABASE, handler); this.db = db; } diff --git metastore/src/java/org/apache/hadoop/hive/metastore/events/PreReadTableEvent.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreReadTableEvent.java similarity index 81% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/PreReadTableEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreReadTableEvent.java index e9d353af28..beec72bc12 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/events/PreReadTableEvent.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreReadTableEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,9 +18,9 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.Table; /** @@ -32,7 +32,7 @@ private final Table table; - public PreReadTableEvent(Table table, HMSHandler handler) { + public PreReadTableEvent(Table table, IHMSHandler handler) { super(PreEventType.READ_TABLE, handler); this.table = table; } diff --git metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AddForeignKeyMessage.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/AddForeignKeyMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AddForeignKeyMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/AddForeignKeyMessage.java index 2eb14a1d61..db8a7d79fb 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AddForeignKeyMessage.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/AddForeignKeyMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AddNotNullConstraintMessage.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/AddNotNullConstraintMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AddNotNullConstraintMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/AddNotNullConstraintMessage.java index 28ee610af8..07fbe644a2 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AddNotNullConstraintMessage.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/AddNotNullConstraintMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AddPartitionMessage.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/AddPartitionMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AddPartitionMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/AddPartitionMessage.java index 774c020ff5..3262b52bea 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AddPartitionMessage.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/AddPartitionMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AddPrimaryKeyMessage.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/AddPrimaryKeyMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AddPrimaryKeyMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/AddPrimaryKeyMessage.java index 0e899ad753..37f2b46c7c 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AddPrimaryKeyMessage.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/AddPrimaryKeyMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AddUniqueConstraintMessage.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/AddUniqueConstraintMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AddUniqueConstraintMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/AddUniqueConstraintMessage.java index 8072d84f63..5c71f22aa7 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AddUniqueConstraintMessage.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/AddUniqueConstraintMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AlterIndexMessage.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/AlterIndexMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AlterIndexMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/AlterIndexMessage.java index e72a94bfbe..78259ba7c3 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AlterIndexMessage.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/AlterIndexMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AlterPartitionMessage.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/AlterPartitionMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AlterPartitionMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/AlterPartitionMessage.java index 077c9f7e9c..aaa7ef5299 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AlterPartitionMessage.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/AlterPartitionMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AlterTableMessage.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/AlterTableMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AlterTableMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/AlterTableMessage.java index 58f01fe126..30e28629a0 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AlterTableMessage.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/AlterTableMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git metastore/src/java/org/apache/hadoop/hive/metastore/messaging/CreateDatabaseMessage.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/CreateDatabaseMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/CreateDatabaseMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/CreateDatabaseMessage.java index 761429884c..328c1180e1 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/messaging/CreateDatabaseMessage.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/CreateDatabaseMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git metastore/src/java/org/apache/hadoop/hive/metastore/messaging/CreateFunctionMessage.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/CreateFunctionMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/CreateFunctionMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/CreateFunctionMessage.java index d94a3f06c3..5478143698 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/messaging/CreateFunctionMessage.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/CreateFunctionMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git metastore/src/java/org/apache/hadoop/hive/metastore/messaging/CreateIndexMessage.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/CreateIndexMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/CreateIndexMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/CreateIndexMessage.java index 3ce0d62c3a..f10710023d 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/messaging/CreateIndexMessage.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/CreateIndexMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git metastore/src/java/org/apache/hadoop/hive/metastore/messaging/CreateTableMessage.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/CreateTableMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/CreateTableMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/CreateTableMessage.java index b75caa6351..49732ff97d 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/messaging/CreateTableMessage.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/CreateTableMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git metastore/src/java/org/apache/hadoop/hive/metastore/messaging/DropConstraintMessage.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/DropConstraintMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/DropConstraintMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/DropConstraintMessage.java index 6e691e9cc4..95c9f9f423 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/messaging/DropConstraintMessage.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/DropConstraintMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git metastore/src/java/org/apache/hadoop/hive/metastore/messaging/DropDatabaseMessage.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/DropDatabaseMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/DropDatabaseMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/DropDatabaseMessage.java index fa6da38f88..a450d47699 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/messaging/DropDatabaseMessage.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/DropDatabaseMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git metastore/src/java/org/apache/hadoop/hive/metastore/messaging/DropFunctionMessage.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/DropFunctionMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/DropFunctionMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/DropFunctionMessage.java index 2b45d40f89..c418feb764 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/messaging/DropFunctionMessage.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/DropFunctionMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git metastore/src/java/org/apache/hadoop/hive/metastore/messaging/DropIndexMessage.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/DropIndexMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/DropIndexMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/DropIndexMessage.java index 5997f92ba9..210b5920f3 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/messaging/DropIndexMessage.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/DropIndexMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git metastore/src/java/org/apache/hadoop/hive/metastore/messaging/DropPartitionMessage.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/DropPartitionMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/DropPartitionMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/DropPartitionMessage.java index d254ad93f6..fa7c9dbe63 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/messaging/DropPartitionMessage.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/DropPartitionMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git metastore/src/java/org/apache/hadoop/hive/metastore/messaging/DropTableMessage.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/DropTableMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/DropTableMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/DropTableMessage.java index 7aed21b488..b8e0e78a9d 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/messaging/DropTableMessage.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/DropTableMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git metastore/src/java/org/apache/hadoop/hive/metastore/messaging/EventMessage.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/EventMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/EventMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/EventMessage.java index 3b37cb3e8b..7b22facb1a 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/messaging/EventMessage.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/EventMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -28,7 +28,7 @@ /** * Enumeration of all supported types of Metastore operations. */ - public static enum EventType { + public enum EventType { CREATE_DATABASE(MessageFactory.CREATE_DATABASE_EVENT), DROP_DATABASE(MessageFactory.DROP_DATABASE_EVENT), diff --git metastore/src/java/org/apache/hadoop/hive/metastore/messaging/InsertMessage.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/InsertMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/InsertMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/InsertMessage.java index 01fc0f2cde..c470097165 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/messaging/InsertMessage.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/InsertMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git metastore/src/java/org/apache/hadoop/hive/metastore/messaging/MessageDeserializer.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/MessageDeserializer.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/MessageDeserializer.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/MessageDeserializer.java index 50a2728aa4..810dc64acd 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/messaging/MessageDeserializer.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/MessageDeserializer.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git metastore/src/java/org/apache/hadoop/hive/metastore/messaging/MessageFactory.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/MessageFactory.java similarity index 87% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/MessageFactory.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/MessageFactory.java index e639e482d0..46fd336789 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/messaging/MessageFactory.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/MessageFactory.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -19,18 +19,20 @@ package org.apache.hadoop.hive.metastore.messaging; -import org.apache.hadoop.hive.common.JavaUtils; -import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.Function; import org.apache.hadoop.hive.metastore.api.Index; +import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.SQLForeignKey; import org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint; import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey; import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint; import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.hadoop.util.ReflectionUtils; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; +import org.apache.hadoop.hive.metastore.utils.JavaUtils; import java.util.Iterator; import java.util.List; @@ -63,21 +65,18 @@ private static MessageFactory instance = null; - protected static final HiveConf hiveConf = new HiveConf(); + protected static final Configuration conf = MetastoreConf.newMetastoreConf(); + /* + // TODO MS-SPLIT I'm 99% certain we don't need this, as MetastoreConf.newMetastoreConf already + adds this resource. static { - hiveConf.addResource("hive-site.xml"); + conf.addResource("hive-site.xml"); } + */ - // This parameter is retained for legacy reasons, in case someone implemented custom - // factories. This, however, should not be the case, since this api was intended to - // be internal-only, and we should manage the jms and json implementations without - // needing this parameter. Marking as deprecated, for removal by 2.4 - see corresponding - // note on the getDeserializer(String,String) method - @Deprecated - private static final String CONF_LABEL_HCAT_MESSAGE_FACTORY_IMPL_PREFIX = "hcatalog.message.factory.impl."; - - protected static final String MS_SERVER_URL = hiveConf.get(HiveConf.ConfVars.METASTOREURIS.name(), ""); - protected static final String MS_SERVICE_PRINCIPAL = hiveConf.get(HiveConf.ConfVars.METASTORE_KERBEROS_PRINCIPAL.name(), ""); + protected static final String MS_SERVER_URL = MetastoreConf.getVar(conf, ConfVars.THRIFT_URIS, ""); + protected static final String MS_SERVICE_PRINCIPAL = + MetastoreConf.getVar(conf, ConfVars.KERBEROS_PRINCIPAL, ""); /** * Getter for MessageFactory instance. @@ -85,17 +84,17 @@ public static MessageFactory getInstance() { if (instance == null) { instance = - getInstance(hiveConf.get(HiveConf.ConfVars.METASTORE_EVENT_MESSAGE_FACTORY.varname)); + getInstance(MetastoreConf.getVar(conf, ConfVars.EVENT_MESSAGE_FACTORY)); } return instance; } private static MessageFactory getInstance(String className) { try { - return (MessageFactory)ReflectionUtils.newInstance(JavaUtils.loadClass(className), hiveConf); + return JavaUtils.newInstance(JavaUtils.getClass(className, MessageFactory.class)); } - catch (ClassNotFoundException classNotFound) { - throw new IllegalStateException("Could not construct MessageFactory implementation: ", classNotFound); + catch (MetaException e) { + throw new IllegalStateException("Could not construct MessageFactory implementation: ", e); } } @@ -107,8 +106,7 @@ private static MessageFactory getInstance(String className) { */ public static MessageDeserializer getDeserializer(String format, String version) { - return getInstance(hiveConf.get(CONF_LABEL_HCAT_MESSAGE_FACTORY_IMPL_PREFIX + format, - HiveConf.ConfVars.METASTORE_EVENT_MESSAGE_FACTORY.varname)).getDeserializer(); + return getInstance(MetastoreConf.getVar(conf, ConfVars.EVENT_MESSAGE_FACTORY)).getDeserializer(); // Note : The reason this method exists outside the no-arg getDeserializer method is in // case there is a user-implemented MessageFactory that's used, and some the messages // are in an older format and the rest in another. Then, what MessageFactory is default diff --git metastore/src/java/org/apache/hadoop/hive/metastore/messaging/PartitionFiles.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/PartitionFiles.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/PartitionFiles.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/PartitionFiles.java index 4fd7f8c57b..308eaf1f64 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/messaging/PartitionFiles.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/PartitionFiles.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddForeignKeyMessage.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddForeignKeyMessage.java similarity index 95% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddForeignKeyMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddForeignKeyMessage.java index 9c04c15191..9c912bf11a 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddForeignKeyMessage.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddForeignKeyMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -51,7 +51,7 @@ public JSONAddForeignKeyMessage(String server, String servicePrincipal, List(); + this.foreignKeyListJson = new ArrayList<>(); try { for (SQLForeignKey pk : fks) { foreignKeyListJson.add(JSONMessageFactory.createForeignKeyObjJson(pk)); @@ -83,7 +83,7 @@ public Long getTimestamp() { @Override public List getForeignKeys() throws Exception { - List fks = new ArrayList(); + List fks = new ArrayList<>(); for (String pkJson : foreignKeyListJson) { fks.add((SQLForeignKey)JSONMessageFactory.getTObj(pkJson, SQLForeignKey.class)); } diff --git metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddNotNullConstraintMessage.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddNotNullConstraintMessage.java similarity index 95% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddNotNullConstraintMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddNotNullConstraintMessage.java index 95ff23adb1..79312e4290 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddNotNullConstraintMessage.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddNotNullConstraintMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -46,7 +46,7 @@ public JSONAddNotNullConstraintMessage(String server, String servicePrincipal, L this.server = server; this.servicePrincipal = servicePrincipal; this.timestamp = timestamp; - this.notNullConstraintListJson = new ArrayList(); + this.notNullConstraintListJson = new ArrayList<>(); try { for (SQLNotNullConstraint nn : nns) { notNullConstraintListJson.add(JSONMessageFactory.createNotNullConstraintObjJson(nn)); @@ -78,7 +78,7 @@ public Long getTimestamp() { @Override public List getNotNullConstraints() throws Exception { - List nns = new ArrayList(); + List nns = new ArrayList<>(); for (String nnJson : notNullConstraintListJson) { nns.add((SQLNotNullConstraint)JSONMessageFactory.getTObj(nnJson, SQLNotNullConstraint.class)); } diff --git metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddPartitionMessage.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddPartitionMessage.java similarity index 97% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddPartitionMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddPartitionMessage.java index db3431eac9..c232ccef3a 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddPartitionMessage.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddPartitionMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -75,8 +75,8 @@ public JSONAddPartitionMessage(String server, String servicePrincipal, Table tab this.table = tableObj.getTableName(); this.tableType = tableObj.getTableType(); this.timestamp = timestamp; - partitions = new ArrayList>(); - partitionListJson = new ArrayList(); + partitions = new ArrayList<>(); + partitionListJson = new ArrayList<>(); Partition partitionObj; try { this.tableObjJson = JSONMessageFactory.createTableObjJson(tableObj); diff --git metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddPrimaryKeyMessage.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddPrimaryKeyMessage.java similarity index 95% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddPrimaryKeyMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddPrimaryKeyMessage.java index 2551cbf006..f4f4d5df99 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddPrimaryKeyMessage.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddPrimaryKeyMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -51,7 +51,7 @@ public JSONAddPrimaryKeyMessage(String server, String servicePrincipal, List(); + this.primaryKeyListJson = new ArrayList<>(); try { for (SQLPrimaryKey pk : pks) { primaryKeyListJson.add(JSONMessageFactory.createPrimaryKeyObjJson(pk)); @@ -83,7 +83,7 @@ public Long getTimestamp() { @Override public List getPrimaryKeys() throws Exception { - List pks = new ArrayList(); + List pks = new ArrayList<>(); for (String pkJson : primaryKeyListJson) { pks.add((SQLPrimaryKey)JSONMessageFactory.getTObj(pkJson, SQLPrimaryKey.class)); } diff --git metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddUniqueConstraintMessage.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddUniqueConstraintMessage.java similarity index 95% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddUniqueConstraintMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddUniqueConstraintMessage.java index 37f6b81372..5a3cf96e6b 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddUniqueConstraintMessage.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddUniqueConstraintMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -48,7 +48,7 @@ public JSONAddUniqueConstraintMessage(String server, String servicePrincipal, Li this.server = server; this.servicePrincipal = servicePrincipal; this.timestamp = timestamp; - this.uniqueConstraintListJson = new ArrayList(); + this.uniqueConstraintListJson = new ArrayList<>(); try { for (SQLUniqueConstraint uk : uks) { uniqueConstraintListJson.add(JSONMessageFactory.createUniqueConstraintObjJson(uk)); @@ -80,7 +80,7 @@ public Long getTimestamp() { @Override public List getUniqueConstraints() throws Exception { - List uks = new ArrayList(); + List uks = new ArrayList<>(); for (String pkJson : uniqueConstraintListJson) { uks.add((SQLUniqueConstraint)JSONMessageFactory.getTObj(pkJson, SQLUniqueConstraint.class)); } diff --git metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterIndexMessage.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterIndexMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterIndexMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterIndexMessage.java index 5dffdd1043..d1657b4669 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterIndexMessage.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterIndexMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterPartitionMessage.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterPartitionMessage.java similarity index 98% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterPartitionMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterPartitionMessage.java index 2991e08836..68cbd95a14 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterPartitionMessage.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterPartitionMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -24,8 +24,6 @@ import org.apache.thrift.TException; import org.codehaus.jackson.annotate.JsonProperty; -import java.util.ArrayList; -import java.util.List; import java.util.Map; /** diff --git metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterTableMessage.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterTableMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterTableMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterTableMessage.java index 1df9c8b159..ce8c0fa18b 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterTableMessage.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterTableMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateDatabaseMessage.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateDatabaseMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateDatabaseMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateDatabaseMessage.java index f8717b24c1..f442e99923 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateDatabaseMessage.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateDatabaseMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateFunctionMessage.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateFunctionMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateFunctionMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateFunctionMessage.java index 3646d85e83..81f5d25fa5 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateFunctionMessage.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateFunctionMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateIndexMessage.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateIndexMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateIndexMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateIndexMessage.java index a2e2fc02e3..a40e5243fc 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateIndexMessage.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateIndexMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateTableMessage.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateTableMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateTableMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateTableMessage.java index 9dd7a744c1..d894af9b58 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateTableMessage.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateTableMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropConstraintMessage.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropConstraintMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropConstraintMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropConstraintMessage.java index 4d3422b94b..9e5d582cef 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropConstraintMessage.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropConstraintMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropDatabaseMessage.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropDatabaseMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropDatabaseMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropDatabaseMessage.java index be17e6d5ac..cd8bcdc812 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropDatabaseMessage.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropDatabaseMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropFunctionMessage.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropFunctionMessage.java similarity index 98% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropFunctionMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropFunctionMessage.java index b9ee4c4dfd..ca32d161a7 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropFunctionMessage.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropFunctionMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -21,7 +21,6 @@ import org.apache.hadoop.hive.metastore.api.Function; import org.apache.hadoop.hive.metastore.messaging.DropFunctionMessage; -import org.apache.thrift.TException; import org.codehaus.jackson.annotate.JsonProperty; /** diff --git metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropIndexMessage.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropIndexMessage.java similarity index 98% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropIndexMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropIndexMessage.java index e99a31f68b..fb719c2dc2 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropIndexMessage.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropIndexMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -21,7 +21,6 @@ import org.apache.hadoop.hive.metastore.api.Index; import org.apache.hadoop.hive.metastore.messaging.DropIndexMessage; -import org.apache.thrift.TException; import org.codehaus.jackson.annotate.JsonProperty; /** diff --git metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropPartitionMessage.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropPartitionMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropPartitionMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropPartitionMessage.java index 576806cf62..4689711693 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropPartitionMessage.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropPartitionMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropTableMessage.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropTableMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropTableMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropTableMessage.java index 4334f732fd..591c9dd088 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropTableMessage.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropTableMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONInsertMessage.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONInsertMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONInsertMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONInsertMessage.java index 1369fd2f3b..39372bdbfa 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONInsertMessage.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONInsertMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageDeserializer.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageDeserializer.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageDeserializer.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageDeserializer.java index 7f588a04c9..15fa4aa039 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageDeserializer.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageDeserializer.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageFactory.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageFactory.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageFactory.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageFactory.java index c7877ee7cc..916a8e8f35 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageFactory.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageFactory.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -205,7 +205,7 @@ private long now() { } static Map getPartitionKeyValues(Table table, Partition partition) { - Map partitionKeys = new LinkedHashMap(); + Map partitionKeys = new LinkedHashMap<>(); for (int i = 0; i < table.getPartitionKeysSize(); ++i) partitionKeys.put(table.getPartitionKeys().get(i).getName(), partition.getValues().get(i)); return partitionKeys; diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/AcidCompactionHistoryService.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/AcidCompactionHistoryService.java new file mode 100644 index 0000000000..97bff0e9b3 --- /dev/null +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/AcidCompactionHistoryService.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.txn; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.RunnableConfigurable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Purges obsolete items from compaction history data + */ +public class AcidCompactionHistoryService implements RunnableConfigurable { + private static final Logger LOG = LoggerFactory.getLogger(AcidCompactionHistoryService.class); + + private Configuration conf; + private TxnStore txnHandler; + + @Override + public void setConf(Configuration configuration) { + this.conf = configuration; + txnHandler = TxnUtils.getTxnStore(conf); + } + + @Override + public Configuration getConf() { + return conf; + } + + @Override + public void run() { + TxnStore.MutexAPI.LockHandle handle = null; + try { + handle = txnHandler.getMutexAPI().acquireLock(TxnStore.MUTEX_KEY.CompactionHistory.name()); + long startTime = System.currentTimeMillis(); + txnHandler.purgeCompactionHistory(); + LOG.debug("History reaper reaper ran for " + (System.currentTimeMillis() - startTime)/1000 + + "seconds."); + } catch(Throwable t) { + LOG.error("Serious error in {}", Thread.currentThread().getName(), ": {}" + t.getMessage(), t); + } finally { + if(handle != null) { + handle.releaseLocks(); + } + } + } +} diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/AcidHouseKeeperService.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/AcidHouseKeeperService.java new file mode 100644 index 0000000000..7450a2f3ac --- /dev/null +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/AcidHouseKeeperService.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.txn; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.RunnableConfigurable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Performs background tasks for Transaction management in Hive. + * Runs inside Hive Metastore Service. + */ +public class AcidHouseKeeperService implements RunnableConfigurable { + private static final Logger LOG = LoggerFactory.getLogger(AcidHouseKeeperService.class); + + private Configuration conf; + private TxnStore txnHandler; + + @Override + public void setConf(Configuration configuration) { + this.conf = configuration; + txnHandler = TxnUtils.getTxnStore(conf); + } + + @Override + public Configuration getConf() { + return conf; + } + + @Override + public void run() { + TxnStore.MutexAPI.LockHandle handle = null; + try { + handle = txnHandler.getMutexAPI().acquireLock(TxnStore.MUTEX_KEY.HouseKeeper.name()); + long startTime = System.currentTimeMillis(); + txnHandler.performTimeOuts(); + LOG.debug("timeout reaper ran for " + (System.currentTimeMillis() - startTime)/1000 + + "seconds."); + } catch(Throwable t) { + LOG.error("Serious error in {}", Thread.currentThread().getName(), ": {}" + t.getMessage(), t); + } finally { + if(handle != null) { + handle.releaseLocks(); + } + } + } +} diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/AcidWriteSetService.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/AcidWriteSetService.java new file mode 100644 index 0000000000..413fe96a18 --- /dev/null +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/AcidWriteSetService.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.txn; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.RunnableConfigurable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Periodically cleans WriteSet tracking information used in Transaction management + */ +public class AcidWriteSetService implements RunnableConfigurable { + private static final Logger LOG = LoggerFactory.getLogger(AcidWriteSetService.class); + + private Configuration conf; + private TxnStore txnHandler; + + @Override + public void setConf(Configuration configuration) { + this.conf = configuration; + txnHandler = TxnUtils.getTxnStore(conf); + } + + @Override + public Configuration getConf() { + return conf; + } + + @Override + public void run() { + TxnStore.MutexAPI.LockHandle handle = null; + try { + handle = txnHandler.getMutexAPI().acquireLock(TxnStore.MUTEX_KEY.WriteSetCleaner.name()); + long startTime = System.currentTimeMillis(); + txnHandler.performWriteSetGC(); + LOG.debug("cleaner ran for " + (System.currentTimeMillis() - startTime)/1000 + "seconds."); + } catch(Throwable t) { + LOG.error("Serious error in {}", Thread.currentThread().getName(), ": {}" + t.getMessage(), t); + } finally { + if(handle != null) { + handle.releaseLocks(); + } + } + } +} diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java index d09c9580ff..756cb4cec2 100644 --- standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java @@ -172,6 +172,12 @@ public static void prepDb(Configuration conf) throws Exception { LOG.error("Error rolling back: " + re.getMessage()); } + // Another thread might have already created these tables. + if (e.getMessage() != null && e.getMessage().contains("already exists")) { + LOG.info("Txn tables already exist, returning"); + return; + } + // This might be a deadlock, if so, let's retry if (e instanceof SQLTransactionRollbackException && deadlockCnt++ < 5) { LOG.warn("Caught deadlock, retrying db creation"); diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/FileUtils.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/FileUtils.java index da0ee80a9e..2dac899d91 100644 --- standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/FileUtils.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/FileUtils.java @@ -363,4 +363,20 @@ public static String makePartName(List partCols, List vals, } return name.toString(); } + + /** + * Determine if two objects reference the same file system. + * @param fs1 first file system + * @param fs2 second file system + * @return return true if both file system arguments point to same file system + */ + public static boolean equalsFileSystem(FileSystem fs1, FileSystem fs2) { + //When file system cache is disabled, you get different FileSystem objects + // for same file system, so '==' can't be used in such cases + //FileSystem api doesn't have a .equals() function implemented, so using + //the uri for comparison. FileSystem already uses uri+Configuration for + //equality in its CACHE . + //Once equality has been added in HDFS-9159, we should make use of it + return fs1.getUri().equals(fs2.getUri()); + } } diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/HdfsUtils.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/HdfsUtils.java index c10e36f94a..ecbddc3e7f 100644 --- standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/HdfsUtils.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/HdfsUtils.java @@ -24,6 +24,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.client.HdfsAdmin; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; @@ -44,6 +45,10 @@ public class HdfsUtils { private static final Logger LOG = LoggerFactory.getLogger(HdfsUtils.class); private static final String DISTCP_OPTIONS_PREFIX = "distcp.options."; + // TODO: this relies on HDFS not changing the format; we assume if we could get inode ID, this + // is still going to work. Otherwise, file IDs can be turned off. Later, we should use + // as public utility method in HDFS to obtain the inode-based path. + private static final String HDFS_ID_PATH_PREFIX = "/.reserved/.inodes/"; /** * Check the permissions on a file. @@ -197,4 +202,20 @@ public static boolean runDistCp(List srcPaths, Path dst, Configuration con return params; } + public static Path getFileIdPath( + FileSystem fileSystem, Path path, long fileId) { + return (fileSystem instanceof DistributedFileSystem) + ? new Path(HDFS_ID_PATH_PREFIX + fileId) : path; + } + + public static long getFileId(FileSystem fs, String path) throws IOException { + return ensureDfs(fs).getClient().getFileInfo(path).getFileId(); + } + + private static DistributedFileSystem ensureDfs(FileSystem fs) { + if (!(fs instanceof DistributedFileSystem)) { + throw new UnsupportedOperationException("Only supported for DFS; got " + fs.getClass()); + } + return (DistributedFileSystem)fs; + } } diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/JavaUtils.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/JavaUtils.java index 593dee3996..b08d9fd71f 100644 --- standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/JavaUtils.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/JavaUtils.java @@ -88,6 +88,20 @@ public static ClassLoader getClassLoader() { } /** + * Create an object of the given class using a no-args constructor + * @param theClass class to return new object of + * @param the type of the class to be returned + * @return an object of the requested type + */ + public static T newInstance(Class theClass) { + try { + return theClass.newInstance(); + } catch (InstantiationException|IllegalAccessException e) { + throw new RuntimeException("Unable to instantiate " + theClass.getName(), e); + } + } + + /** * @return name of current host */ public static String hostname() { diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java index 77790adfff..6e71019bb6 100644 --- standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java @@ -21,20 +21,32 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.commons.collections.ListUtils; import org.apache.commons.lang.*; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.hive.common.StatsSetupConst; +import org.apache.hadoop.hive.metastore.ColumnType; +import org.apache.hadoop.hive.metastore.TableType; +import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.ColumnStatistics; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; +import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.Decimal; +import org.apache.hadoop.hive.metastore.api.EnvironmentContext; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.Order; +import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.SerDeInfo; import org.apache.hadoop.hive.metastore.api.SkewedInfo; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.hive.metastore.columnstats.aggr.ColumnStatsAggregator; import org.apache.hadoop.hive.metastore.columnstats.aggr.ColumnStatsAggregatorFactory; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -48,9 +60,11 @@ import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.SortedMap; import java.util.SortedSet; import java.util.TreeMap; @@ -60,6 +74,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.regex.Matcher; import java.util.regex.Pattern; public class MetaStoreUtils { @@ -73,9 +88,20 @@ protected DateFormat initialValue() { return val; } }; + // Indicates a type was derived from the deserializer rather than Hive's metadata. + public static final String TYPE_FROM_DESERIALIZER = ""; + private static final Charset ENCODING = StandardCharsets.UTF_8; private static final Logger LOG = LoggerFactory.getLogger(MetaStoreUtils.class); + // Right now we only support one special character '/'. + // More special characters can be added accordingly in the future. + // NOTE: + // If the following array is updated, please also be sure to update the + // configuration parameter documentation + // HIVE_SUPPORT_SPECICAL_CHARACTERS_IN_TABLE_NAMES in HiveConf as well. + private static final char[] specialCharactersInTableNames = new char[] { '/' }; + /** * Catches exceptions that can't be handled and bundles them to MetaException * @@ -139,8 +165,9 @@ public static MetaException newMetaException(String errorMessage, Exception e) { return org.apache.commons.lang.StringUtils.defaultString(string); } }; + /** - * We have aneed to sanity-check the map before conversion from persisted objects to + * We have a need to sanity-check the map before conversion from persisted objects to * metadata thrift objects because null values in maps will cause a NPE if we send * across thrift. Pruning is appropriate for most cases except for databases such as * Oracle where Empty strings are stored as nulls, in which case we need to handle that. @@ -359,4 +386,365 @@ public static String getPartitionValWithInvalidCharacter(List partVals, } return colNames; } + + /** + * validateName + * + * Checks the name conforms to our standars which are: "[a-zA-z_0-9]+". checks + * this is just characters and numbers and _ + * + * @param name + * the name to validate + * @param conf + * hive configuration + * @return true or false depending on conformance + * if it doesn't match the pattern. + */ + public static boolean validateName(String name, Configuration conf) { + Pattern tpat = null; + String allowedCharacters = "\\w_"; + if (conf != null + && MetastoreConf.getBoolVar(conf, + MetastoreConf.ConfVars.SUPPORT_SPECICAL_CHARACTERS_IN_TABLE_NAMES)) { + for (Character c : specialCharactersInTableNames) { + allowedCharacters += c; + } + } + tpat = Pattern.compile("[" + allowedCharacters + "]+"); + Matcher m = tpat.matcher(name); + return m.matches(); + } + + /* + * At the Metadata level there are no restrictions on Column Names. + */ + public static boolean validateColumnName(String name) { + return true; + } + + static public String validateTblColumns(List cols) { + for (FieldSchema fieldSchema : cols) { + // skip this, as validateColumnName always returns true + /* + if (!validateColumnName(fieldSchema.getName())) { + return "name: " + fieldSchema.getName(); + } + */ + String typeError = validateColumnType(fieldSchema.getType()); + if (typeError != null) { + return typeError; + } + } + return null; + } + + private static String validateColumnType(String type) { + if (type.equals(TYPE_FROM_DESERIALIZER)) return null; + int last = 0; + boolean lastAlphaDigit = isValidTypeChar(type.charAt(last)); + for (int i = 1; i <= type.length(); i++) { + if (i == type.length() + || isValidTypeChar(type.charAt(i)) != lastAlphaDigit) { + String token = type.substring(last, i); + last = i; + if (!ColumnType.AllTypes.contains(token)) { + return "type: " + type; + } + break; + } + } + return null; + } + + private static boolean isValidTypeChar(char c) { + return Character.isLetterOrDigit(c) || c == '_'; + } + + /** + * Determines whether a table is an external table. + * + * @param table table of interest + * + * @return true if external + */ + public static boolean isExternalTable(Table table) { + if (table == null) { + return false; + } + Map params = table.getParameters(); + if (params == null) { + return false; + } + + return "TRUE".equalsIgnoreCase(params.get("EXTERNAL")); + } + + // check if stats need to be (re)calculated + public static boolean requireCalStats(Configuration hiveConf, Partition oldPart, + Partition newPart, Table tbl, EnvironmentContext environmentContext) { + + if (environmentContext != null + && environmentContext.isSetProperties() + && StatsSetupConst.TRUE.equals(environmentContext.getProperties().get( + StatsSetupConst.DO_NOT_UPDATE_STATS))) { + return false; + } + + if (MetaStoreUtils.isView(tbl)) { + return false; + } + + if (oldPart == null && newPart == null) { + return true; + } + + // requires to calculate stats if new partition doesn't have it + if ((newPart == null) || (newPart.getParameters() == null) + || !containsAllFastStats(newPart.getParameters())) { + return true; + } + + if (environmentContext != null && environmentContext.isSetProperties()) { + String statsType = environmentContext.getProperties().get(StatsSetupConst.STATS_GENERATED); + // no matter STATS_GENERATED is USER or TASK, all need to re-calculate the stats: + // USER: alter table .. update statistics + // TASK: from some sql operation which could collect and compute stats + if (StatsSetupConst.TASK.equals(statsType) || StatsSetupConst.USER.equals(statsType)) { + return true; + } + } + + // requires to calculate stats if new and old have different fast stats + return !isFastStatsSame(oldPart, newPart); + } + + public static boolean isView(Table table) { + if (table == null) { + return false; + } + return TableType.VIRTUAL_VIEW.toString().equals(table.getTableType()); + } + + /** + * @param partParams + * @return True if the passed Parameters Map contains values for all "Fast Stats". + */ + private static boolean containsAllFastStats(Map partParams) { + for (String stat : StatsSetupConst.fastStats) { + if (!partParams.containsKey(stat)) { + return false; + } + } + return true; + } + + public static boolean isFastStatsSame(Partition oldPart, Partition newPart) { + // requires to calculate stats if new and old have different fast stats + if ((oldPart != null) && (oldPart.getParameters() != null)) { + for (String stat : StatsSetupConst.fastStats) { + if (oldPart.getParameters().containsKey(stat)) { + Long oldStat = Long.parseLong(oldPart.getParameters().get(stat)); + Long newStat = Long.parseLong(newPart.getParameters().get(stat)); + if (!oldStat.equals(newStat)) { + return false; + } + } else { + return false; + } + } + return true; + } + return false; + } + + public static boolean updateTableStatsFast(Database db, Table tbl, Warehouse wh, + boolean madeDir, EnvironmentContext environmentContext) throws MetaException { + return updateTableStatsFast(db, tbl, wh, madeDir, false, environmentContext); + } + + public static boolean updateTableStatsFast(Database db, Table tbl, Warehouse wh, + boolean madeDir, boolean forceRecompute, EnvironmentContext environmentContext) throws MetaException { + if (tbl.getPartitionKeysSize() == 0) { + // Update stats only when unpartitioned + FileStatus[] fileStatuses = wh.getFileStatusesForUnpartitionedTable(db, tbl); + return updateTableStatsFast(tbl, fileStatuses, madeDir, forceRecompute, environmentContext); + } else { + return false; + } + } + + /** + * Updates the numFiles and totalSize parameters for the passed Table by querying + * the warehouse if the passed Table does not already have values for these parameters. + * @param tbl + * @param fileStatus + * @param newDir if true, the directory was just created and can be assumed to be empty + * @param forceRecompute Recompute stats even if the passed Table already has + * these parameters set + * @return true if the stats were updated, false otherwise + */ + public static boolean updateTableStatsFast(Table tbl, FileStatus[] fileStatus, boolean newDir, + boolean forceRecompute, EnvironmentContext environmentContext) throws MetaException { + + Map params = tbl.getParameters(); + + if ((params!=null) && params.containsKey(StatsSetupConst.DO_NOT_UPDATE_STATS)){ + boolean doNotUpdateStats = Boolean.valueOf(params.get(StatsSetupConst.DO_NOT_UPDATE_STATS)); + params.remove(StatsSetupConst.DO_NOT_UPDATE_STATS); + tbl.setParameters(params); // to make sure we remove this marker property + if (doNotUpdateStats){ + return false; + } + } + + boolean updated = false; + if (forceRecompute || + params == null || + !containsAllFastStats(params)) { + if (params == null) { + params = new HashMap(); + } + if (!newDir) { + // The table location already exists and may contain data. + // Let's try to populate those stats that don't require full scan. + LOG.info("Updating table stats fast for " + tbl.getTableName()); + populateQuickStats(fileStatus, params); + LOG.info("Updated size of table " + tbl.getTableName() +" to "+ params.get(StatsSetupConst.TOTAL_SIZE)); + if (environmentContext != null + && environmentContext.isSetProperties() + && StatsSetupConst.TASK.equals(environmentContext.getProperties().get( + StatsSetupConst.STATS_GENERATED))) { + StatsSetupConst.setBasicStatsState(params, StatsSetupConst.TRUE); + } else { + StatsSetupConst.setBasicStatsState(params, StatsSetupConst.FALSE); + } + } + tbl.setParameters(params); + updated = true; + } + return updated; + } + + public static void populateQuickStats(FileStatus[] fileStatus, Map params) { + int numFiles = 0; + long tableSize = 0L; + for (FileStatus status : fileStatus) { + // don't take directories into account for quick stats + if (!status.isDir()) { + tableSize += status.getLen(); + numFiles += 1; + } + } + params.put(StatsSetupConst.NUM_FILES, Integer.toString(numFiles)); + params.put(StatsSetupConst.TOTAL_SIZE, Long.toString(tableSize)); + } + + public static boolean areSameColumns(List oldCols, List newCols) { + return ListUtils.isEqualList(oldCols, newCols); + } + + public static void updateBasicState(EnvironmentContext environmentContext, Map + params) { + if (params == null) { + return; + } + if (environmentContext != null + && environmentContext.isSetProperties() + && StatsSetupConst.TASK.equals(environmentContext.getProperties().get( + StatsSetupConst.STATS_GENERATED))) { + StatsSetupConst.setBasicStatsState(params, StatsSetupConst.TRUE); + } else { + StatsSetupConst.setBasicStatsState(params, StatsSetupConst.FALSE); + } + } + + public static boolean updatePartitionStatsFast(Partition part, Warehouse wh, EnvironmentContext environmentContext) + throws MetaException { + return updatePartitionStatsFast(part, wh, false, false, environmentContext); + } + + public static boolean updatePartitionStatsFast(Partition part, Warehouse wh, boolean madeDir, EnvironmentContext environmentContext) + throws MetaException { + return updatePartitionStatsFast(part, wh, madeDir, false, environmentContext); + } + + /** + * Updates the numFiles and totalSize parameters for the passed Partition by querying + * the warehouse if the passed Partition does not already have values for these parameters. + * @param part + * @param wh + * @param madeDir if true, the directory was just created and can be assumed to be empty + * @param forceRecompute Recompute stats even if the passed Partition already has + * these parameters set + * @return true if the stats were updated, false otherwise + */ + public static boolean updatePartitionStatsFast(Partition part, Warehouse wh, + boolean madeDir, boolean forceRecompute, EnvironmentContext environmentContext) throws MetaException { + return updatePartitionStatsFast(new PartitionSpecProxy.SimplePartitionWrapperIterator(part), + wh, madeDir, forceRecompute, environmentContext); + } + /** + * Updates the numFiles and totalSize parameters for the passed Partition by querying + * the warehouse if the passed Partition does not already have values for these parameters. + * @param part + * @param wh + * @param madeDir if true, the directory was just created and can be assumed to be empty + * @param forceRecompute Recompute stats even if the passed Partition already has + * these parameters set + * @return true if the stats were updated, false otherwise + */ + public static boolean updatePartitionStatsFast(PartitionSpecProxy.PartitionIterator part, Warehouse wh, + boolean madeDir, boolean forceRecompute, EnvironmentContext environmentContext) throws MetaException { + Map params = part.getParameters(); + boolean updated = false; + if (forceRecompute || + params == null || + !containsAllFastStats(params)) { + if (params == null) { + params = new HashMap(); + } + if (!madeDir) { + // The partition location already existed and may contain data. Lets try to + // populate those statistics that don't require a full scan of the data. + LOG.warn("Updating partition stats fast for: " + part.getTableName()); + FileStatus[] fileStatus = wh.getFileStatusesForLocation(part.getLocation()); + populateQuickStats(fileStatus, params); + LOG.warn("Updated size to " + params.get(StatsSetupConst.TOTAL_SIZE)); + updateBasicState(environmentContext, params); + } + part.setParameters(params); + updated = true; + } + return updated; + } + + /* + * This method is to check if the new column list includes all the old columns with same name and + * type. The column comment does not count. + */ + public static boolean columnsIncludedByNameType(List oldCols, + List newCols) { + if (oldCols.size() > newCols.size()) { + return false; + } + + Map columnNameTypePairMap = new HashMap(newCols.size()); + for (FieldSchema newCol : newCols) { + columnNameTypePairMap.put(newCol.getName().toLowerCase(), newCol.getType()); + } + for (final FieldSchema oldCol : oldCols) { + if (!columnNameTypePairMap.containsKey(oldCol.getName()) + || !columnNameTypePairMap.get(oldCol.getName()).equalsIgnoreCase(oldCol.getType())) { + return false; + } + } + + return true; + } + + /** Duplicates AcidUtils; used in a couple places in metastore. */ + public static boolean isInsertOnlyTableParam(Map params) { + String transactionalProp = params.get(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES); + return (transactionalProp != null && "insert_only".equalsIgnoreCase(transactionalProp)); + } } diff --git metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveAlterHandler.java standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestHiveAlterHandler.java similarity index 100% rename from metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveAlterHandler.java rename to standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestHiveAlterHandler.java diff --git metastore/src/test/org/apache/hadoop/hive/metastore/TestRetriesInRetryingHMSHandler.java standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestRetriesInRetryingHMSHandler.java similarity index 67% rename from metastore/src/test/org/apache/hadoop/hive/metastore/TestRetriesInRetryingHMSHandler.java rename to standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestRetriesInRetryingHMSHandler.java index 0d2a9cb180..8d44bf8081 100644 --- metastore/src/test/org/apache/hadoop/hive/metastore/TestRetriesInRetryingHMSHandler.java +++ standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestRetriesInRetryingHMSHandler.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -24,26 +24,25 @@ import javax.jdo.JDOException; -import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; import org.junit.BeforeClass; import org.junit.Test; import org.mockito.Mockito; public class TestRetriesInRetryingHMSHandler { - private static HiveConf hiveConf; + private static Configuration conf; private static final int RETRY_ATTEMPTS = 3; @BeforeClass public static void setup() throws IOException { - hiveConf = new HiveConf(); - int port = MetaStoreTestUtils.findFreePort(); - hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); - hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); - hiveConf.setIntVar(HiveConf.ConfVars.HMSHANDLERATTEMPTS, RETRY_ATTEMPTS); - hiveConf.setTimeVar(HiveConf.ConfVars.HMSHANDLERINTERVAL, 10, TimeUnit.MILLISECONDS); - hiveConf.setBoolVar(HiveConf.ConfVars.HMSHANDLERFORCERELOADCONF, false); + conf = MetastoreConf.newMetastoreConf(); + MetastoreConf.setLongVar(conf, ConfVars.HMSHANDLERATTEMPTS, RETRY_ATTEMPTS); + MetastoreConf.setTimeVar(conf, ConfVars.HMSHANDLERINTERVAL, 10, TimeUnit.MILLISECONDS); + MetastoreConf.setBoolVar(conf, ConfVars.HMSHANDLERFORCERELOADCONF, false); } /* @@ -52,13 +51,13 @@ public static void setup() throws IOException { */ @Test public void testRetryInit() throws MetaException { - IHMSHandler mockBaseHandler = Mockito.mock(HiveMetaStore.HMSHandler.class); - Mockito.when(mockBaseHandler.getConf()).thenReturn(hiveConf); + IHMSHandler mockBaseHandler = Mockito.mock(IHMSHandler.class); + Mockito.when(mockBaseHandler.getConf()).thenReturn(conf); Mockito .doThrow(JDOException.class) .doNothing() .when(mockBaseHandler).init(); - RetryingHMSHandler.getProxy(hiveConf, mockBaseHandler, false); + RetryingHMSHandler.getProxy(conf, mockBaseHandler, false); Mockito.verify(mockBaseHandler, Mockito.times(2)).init(); } @@ -67,10 +66,10 @@ public void testRetryInit() throws MetaException { */ @Test public void testNoRetryInit() throws MetaException { - IHMSHandler mockBaseHandler = Mockito.mock(HiveMetaStore.HMSHandler.class); - Mockito.when(mockBaseHandler.getConf()).thenReturn(hiveConf); + IHMSHandler mockBaseHandler = Mockito.mock(IHMSHandler.class); + Mockito.when(mockBaseHandler.getConf()).thenReturn(conf); Mockito.doNothing().when(mockBaseHandler).init(); - RetryingHMSHandler.getProxy(hiveConf, mockBaseHandler, false); + RetryingHMSHandler.getProxy(conf, mockBaseHandler, false); Mockito.verify(mockBaseHandler, Mockito.times(1)).init(); } @@ -80,10 +79,10 @@ public void testNoRetryInit() throws MetaException { */ @Test(expected = MetaException.class) public void testRetriesLimit() throws MetaException { - IHMSHandler mockBaseHandler = Mockito.mock(HiveMetaStore.HMSHandler.class); - Mockito.when(mockBaseHandler.getConf()).thenReturn(hiveConf); + IHMSHandler mockBaseHandler = Mockito.mock(IHMSHandler.class); + Mockito.when(mockBaseHandler.getConf()).thenReturn(conf); Mockito.doThrow(JDOException.class).when(mockBaseHandler).init(); - RetryingHMSHandler.getProxy(hiveConf, mockBaseHandler, false); + RetryingHMSHandler.getProxy(conf, mockBaseHandler, false); Mockito.verify(mockBaseHandler, Mockito.times(RETRY_ATTEMPTS)).init(); } @@ -93,8 +92,8 @@ public void testRetriesLimit() throws MetaException { */ @Test public void testWrappedMetaExceptionRetry() throws MetaException { - IHMSHandler mockBaseHandler = Mockito.mock(HiveMetaStore.HMSHandler.class); - Mockito.when(mockBaseHandler.getConf()).thenReturn(hiveConf); + IHMSHandler mockBaseHandler = Mockito.mock(IHMSHandler.class); + Mockito.when(mockBaseHandler.getConf()).thenReturn(conf); //JDOException wrapped in MetaException wrapped in InvocationException MetaException me = new MetaException("Dummy exception"); me.initCause(new JDOException()); @@ -103,7 +102,7 @@ public void testWrappedMetaExceptionRetry() throws MetaException { .doThrow(me) .doNothing() .when(mockBaseHandler).init(); - RetryingHMSHandler.getProxy(hiveConf, mockBaseHandler, false); + RetryingHMSHandler.getProxy(conf, mockBaseHandler, false); Mockito.verify(mockBaseHandler, Mockito.times(2)).init(); } } diff --git metastore/src/test/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageDeserializerTest.java standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageDeserializerTest.java similarity index 99% rename from metastore/src/test/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageDeserializerTest.java rename to standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageDeserializerTest.java index c278338c19..90329c8d40 100644 --- metastore/src/test/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageDeserializerTest.java +++ standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageDeserializerTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreUtils.java standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/utils/TestMetaStoreUtils.java similarity index 50% rename from itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreUtils.java rename to standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/utils/TestMetaStoreUtils.java index c4dd97ec6c..32ad63a728 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreUtils.java +++ standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/utils/TestMetaStoreUtils.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -16,17 +16,23 @@ * limitations under the License. */ -package org.apache.hadoop.hive.metastore; +package org.apache.hadoop.hive.metastore.utils; -import junit.framework.TestCase; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.junit.Assert; +import org.junit.Test; +import java.util.Arrays; import java.util.HashMap; import java.util.Map; -public class TestMetaStoreUtils extends TestCase { +import static org.junit.Assert.assertEquals; +public class TestMetaStoreUtils { + + @Test public void testTrimMapNullsXform() throws Exception { - Map m = new HashMap(); + Map m = new HashMap<>(); m.put("akey","aval"); m.put("blank",""); m.put("null",null); @@ -41,8 +47,9 @@ public void testTrimMapNullsXform() throws Exception { assertEquals("",xformed.get("null")); } + @Test public void testTrimMapNullsPrune() throws Exception { - Map m = new HashMap(); + Map m = new HashMap<>(); m.put("akey","aval"); m.put("blank",""); m.put("null",null); @@ -57,6 +64,21 @@ public void testTrimMapNullsPrune() throws Exception { assert(!pruned.containsValue(null)); } + @Test + public void testcolumnsIncludedByNameType() { + FieldSchema col1 = new FieldSchema("col1", "string", "col1 comment"); + FieldSchema col1a = new FieldSchema("col1", "string", "col1 but with a different comment"); + FieldSchema col2 = new FieldSchema("col2", "string", "col2 comment"); + FieldSchema col3 = new FieldSchema("col3", "string", "col3 comment"); + Assert.assertTrue(org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.columnsIncludedByNameType(Arrays.asList(col1), Arrays.asList(col1))); + Assert.assertTrue(org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.columnsIncludedByNameType(Arrays.asList(col1), Arrays.asList(col1a))); + Assert.assertTrue(org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.columnsIncludedByNameType(Arrays.asList(col1, col2), Arrays.asList(col1, col2))); + Assert.assertTrue(org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.columnsIncludedByNameType(Arrays.asList(col1, col2), Arrays.asList(col2, col1))); + Assert.assertTrue(org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.columnsIncludedByNameType(Arrays.asList(col1, col2), Arrays.asList(col1, col2, col3))); + Assert.assertTrue(org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.columnsIncludedByNameType(Arrays.asList(col1, col2), Arrays.asList(col3, col2, col1))); + Assert.assertFalse(org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.columnsIncludedByNameType(Arrays.asList(col1, col2), Arrays.asList(col1))); + } + }