diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 3ab1dba..c0c192b 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -400,6 +400,9 @@ public void setSparkConfigUpdated(boolean isSparkConfigUpdated) { HADOOPNUMREDUCERS("mapreduce.job.reduces", -1, "", true), // Metastore stuff. Be sure to update HiveConf.metaVars when you add something here! + IMETASTORECLIENTFACTORY("hive.imetastoreclient.factory", + "org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClientFactory", + "The name of the factory class that produces objects implementing the IMetaStoreClient interface."), METASTOREWAREHOUSE("hive.metastore.warehouse.dir", "/user/hive/warehouse", "location of default database for the warehouse"), METASTOREURIS("hive.metastore.uris", "", diff --git ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index f51e1a4..6141f61 100644 --- ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -18,7 +18,6 @@ package org.apache.hadoop.hive.ql.metadata; -import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE; import static org.apache.hadoop.hive.serde.serdeConstants.COLLECTION_DELIM; import static org.apache.hadoop.hive.serde.serdeConstants.ESCAPE_CHAR; import static org.apache.hadoop.hive.serde.serdeConstants.FIELD_DELIM; @@ -44,7 +43,6 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; -import com.google.common.collect.ImmutableMap; import javax.jdo.JDODataStoreException; import org.apache.hadoop.conf.Configuration; @@ -60,13 +58,10 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.HiveMetaException; -import org.apache.hadoop.hive.metastore.HiveMetaHook; -import org.apache.hadoop.hive.metastore.HiveMetaHookLoader; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.metastore.MetaStoreUtils; import org.apache.hadoop.hive.metastore.PartitionDropOptions; -import org.apache.hadoop.hive.metastore.RetryingMetaStoreClient; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.AggrStats; @@ -125,10 +120,12 @@ import org.apache.hadoop.mapred.InputFormat; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.StringUtils; +import org.apache.hive.common.util.ReflectionUtil; import org.apache.thrift.TException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Sets; /** @@ -146,6 +143,7 @@ static final private Logger LOG = LoggerFactory.getLogger("hive.ql.metadata.Hive"); private HiveConf conf = null; + private String metaStoreClientFactory; private IMetaStoreClient metaStoreClient; private UserGroupInformation owner; @@ -335,6 +333,7 @@ public static void closeCurrent() { */ private Hive(HiveConf c, boolean doRegisterAllFns) throws HiveException { conf = c; + metaStoreClientFactory = conf.getVar(HiveConf.ConfVars.IMETASTORECLIENTFACTORY); if (doRegisterAllFns) { registerAllFunctionsOnce(); } @@ -3094,39 +3093,18 @@ public static boolean isHadoop1() { */ private IMetaStoreClient createMetaStoreClient(boolean allowEmbedded) throws MetaException { - HiveMetaHookLoader hookLoader = new HiveMetaHookLoader() { - @Override - public HiveMetaHook getHook( - org.apache.hadoop.hive.metastore.api.Table tbl) - throws MetaException { - - try { - if (tbl == null) { - return null; - } - HiveStorageHandler storageHandler = - HiveUtils.getStorageHandler(conf, - tbl.getParameters().get(META_TABLE_STORAGE)); - if (storageHandler == null) { - return null; - } - return storageHandler.getMetaHook(); - } catch (HiveException ex) { - LOG.error(StringUtils.stringifyException(ex)); - throw new MetaException( - "Failed to load storage handler: " + ex.getMessage()); - } - } - }; - - if (conf.getBoolVar(ConfVars.METASTORE_FASTPATH)) { - return new SessionHiveMetaStoreClient(conf, hookLoader, allowEmbedded); - } else { - return RetryingMetaStoreClient.getProxy(conf, hookLoader, metaCallTimeMap, - SessionHiveMetaStoreClient.class.getName(), allowEmbedded); + HiveMetaStoreClientFactory factory; + try { + // IMetaStoreClient is not Configurable so pass "null" for Configuration + factory = (HiveMetaStoreClientFactory) ReflectionUtil.newInstance(conf.getClassByName(metaStoreClientFactory), null); + } catch (Exception e) { + String errorMessage = "Unable to instantiate a metastore client factory " + metaStoreClientFactory + ": " + e; + LOG.error(errorMessage, e); + throw new MetaException(errorMessage); } + return factory.createMetaStoreClient(conf, allowEmbedded, metaCallTimeMap); } - + public static class SchemaException extends MetaException { private static final long serialVersionUID = 1L; public SchemaException(String message) { diff --git ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMetaStoreClientFactory.java ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMetaStoreClientFactory.java new file mode 100644 index 0000000..f10307c --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMetaStoreClientFactory.java @@ -0,0 +1,32 @@ +package org.apache.hadoop.hive.ql.metadata; + +import java.util.Map; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.hadoop.hive.metastore.api.MetaException; + +/** + * Abstract factory that defines an interface for other factories that produce concrete + * MetaStoreClient objects. + * + * @author Austin Lee + * + */ +public interface HiveMetaStoreClientFactory { + + /** + * A method for producing IMetaStoreClient objects. + * + * The implementation returned by this method must throw a MetaException if allowEmbedded = true + * and it does not support embedded mode. + * + * @param conf Hive Configuration. + * @param allowEmbedded Flag indicating the implementation must run in-process, e.g. for unit testing or "fast path". + * @param metaCallTimeMap A container for storing entry and exit timestamps of IMetaStoreClient method invocations. + * @return IMetaStoreClient An implementation of IMetaStoreClient. + * @throws MetaException + */ + IMetaStoreClient createMetaStoreClient(HiveConf conf, boolean allowEmbedded, Map metaCallTimeMap) + throws MetaException; +} diff --git ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClientFactory.java ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClientFactory.java new file mode 100644 index 0000000..8c20cbc --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClientFactory.java @@ -0,0 +1,69 @@ +package org.apache.hadoop.hive.ql.metadata; + +import static com.google.common.base.Preconditions.checkNotNull; +import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE; + +import java.util.Map; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.metastore.HiveMetaHook; +import org.apache.hadoop.hive.metastore.HiveMetaHookLoader; +import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.hadoop.hive.metastore.RetryingMetaStoreClient; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.util.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Default MetaStoreClientFactory for Hive which produces SessionHiveMetaStoreClient objects. + * + * @author Austin Lee + * + */ +public class SessionHiveMetaStoreClientFactory implements HiveMetaStoreClientFactory { + + private final static Logger LOG = LoggerFactory.getLogger("hive.ql.metadata.SessionHiveMetaStoreClientFactory"); + + @Override + public IMetaStoreClient createMetaStoreClient(final HiveConf conf, boolean allowEmbedded, Map metaCallTimeMap) + throws MetaException { + + checkNotNull(conf, "conf cannot be null!"); + checkNotNull(metaCallTimeMap, "metaCallTimeMap cannot be null!"); + + HiveMetaHookLoader hookLoader = new HiveMetaHookLoader() { + @Override + public HiveMetaHook getHook( + org.apache.hadoop.hive.metastore.api.Table tbl) + throws MetaException { + + try { + if (tbl == null) { + return null; + } + HiveStorageHandler storageHandler = + HiveUtils.getStorageHandler(conf, + tbl.getParameters().get(META_TABLE_STORAGE)); + if (storageHandler == null) { + return null; + } + return storageHandler.getMetaHook(); + } catch (HiveException ex) { + LOG.error(StringUtils.stringifyException(ex)); + throw new MetaException( + "Failed to load storage handler: " + ex.getMessage()); + } + } + }; + + if (conf.getBoolVar(ConfVars.METASTORE_FASTPATH)) { + return new SessionHiveMetaStoreClient(conf, hookLoader, allowEmbedded); + } else { + return RetryingMetaStoreClient.getProxy(conf, hookLoader, metaCallTimeMap, + SessionHiveMetaStoreClient.class.getName(), allowEmbedded); + } + } + +} diff --git ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java index b96de63..10d6b6d 100755 --- ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java +++ ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java @@ -33,6 +33,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.metastore.MetaStoreUtils; import org.apache.hadoop.hive.metastore.PartitionDropOptions; import org.apache.hadoop.hive.metastore.Warehouse; @@ -825,7 +826,45 @@ public void testHiveRefreshOnConfChange() throws Throwable{ newHiveObj = Hive.get(newHconf); assertTrue(prevHiveObj != newHiveObj); } - + + public void testLoadingHiveMetaStoreClientFactory() throws Throwable { + String factoryName = SessionHiveMetaStoreClientFactory.class.getName(); + HiveConf conf = new HiveConf(); + conf.setVar(ConfVars.IMETASTORECLIENTFACTORY, factoryName); + // Make sure we instantiate the embedded version + // so the implementation chosen is SessionHiveMetaStoreClient, not a retryable version of it. + conf.setBoolVar(ConfVars.METASTORE_FASTPATH, true); + // The current object was constructed in setUp() before we got here + // so clean that up so we can inject our own dummy implementation of IMetaStoreClient + Hive.closeCurrent(); + Hive hive = Hive.get(conf); + IMetaStoreClient tmp = null; + try { + tmp = hive.getMSC(); + assertTrue("getMSC() succeeded.", tmp != null); + assertTrue("Invalid default client implementation created.", tmp instanceof SessionHiveMetaStoreClient); + } catch (Exception e) { + assertTrue("getMSC() failed.", false); + } + } + + public void testLoadingInvalidHiveMetaStoreClientFactory() throws Throwable { + // Intentionally invalid class + String factoryName = String.class.getName(); + HiveConf conf = new HiveConf(); + conf.setVar(HiveConf.ConfVars.IMETASTORECLIENTFACTORY, factoryName); + // The current object was constructed in setUp() before we got here + // so clean that up so we can inject our own dummy implementation of IMetaStoreClient + Hive.closeCurrent(); + Hive hive = Hive.get(conf); + try { + hive.getMSC(); + assertTrue("getMSC() succeeded, which is NOT expected!", false); + } catch (Exception e) { + assertTrue("getMSC() failed, which IS expected.", true); + } + } + // shamelessly copied from Path in hadoop-2 private static final String SEPARATOR = "/"; private static final char SEPARATOR_CHAR = '/';