From 39df0ceef52d4146d05cebd2437130722b13c203 Mon Sep 17 00:00:00 2001 From: Alexander Kolbasov Date: Fri, 10 Aug 2018 14:29:08 -0700 Subject: [PATCH 1/1] HIVE-20194: HiveMetastoreClient should use reflection to instantiate embedded HMS instance --- .../hadoop/hive/metastore/HiveMetaStore.java | 18 ++++++-- .../hadoop/hive/metastore/HiveMetaStoreClient.java | 51 +++++++++++++++++++++- .../metastore/HiveMetaStoreClientPreCatalog.java | 8 ++-- 3 files changed, 69 insertions(+), 8 deletions(-) diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index a53d4be03d..d34e94414f 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -8979,10 +8979,22 @@ private static IHMSHandler newRetryingHMSHandler(IHMSHandler baseHandler, Config return RetryingHMSHandler.getProxy(conf, baseHandler, local); } - static Iface newRetryingHMSHandler(String name, Configuration conf, boolean local) + /** + * Create retrying HMS handler for embedded metastore. + * + *

IMPORTANT

+ * + * This method is called indirectly by HiveMetastoreClient and HiveMetaStoreClientPreCatalog + * using reflection. It can not be removed and its arguments can't be changed without matching + * change in HiveMetastoreClient and HiveMetaStoreClientPreCatalog. + * + * @param conf configuration to use + * @throws MetaException + */ + static Iface newRetryingHMSHandler(Configuration conf) throws MetaException { - HMSHandler baseHandler = new HiveMetaStore.HMSHandler(name, conf, false); - return RetryingHMSHandler.getProxy(conf, baseHandler, local); + HMSHandler baseHandler = new HiveMetaStore.HMSHandler("hive client", conf, false); + return RetryingHMSHandler.getProxy(conf, baseHandler, true); } /** diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java index 91c86a749c..5ae00af564 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java @@ -107,6 +107,14 @@ public final static ClientCapabilities TEST_VERSION = new ClientCapabilities( Lists.newArrayList(ClientCapability.INSERT_ONLY_TABLES, ClientCapability.TEST_CAPABILITY)); + // Name of the HiveMetaStore class. It is used to initialize embedded metastore + private static final String HIVE_METASTORE_CLASS = + "org.apache.hadoop.hive.metastore.HiveMetaStore"; + + // Method used to create Hive Metastore client. It is called as + // HiveMetaStore.newRetryingHMSHandler("hive client", this.conf, true); + private static final String HIVE_METASTORE_CREATE_HANDLER_METHOD = "newRetryingHMSHandler"; + ThriftHiveMetastore.Iface client = null; private TTransport transport = null; private boolean isConnected = false; @@ -164,9 +172,11 @@ public HiveMetaStoreClient(Configuration conf, HiveMetaHookLoader hookLoader, Bo throw new MetaException("Embedded metastore is not allowed here. Please configure " + ConfVars.THRIFT_URIS.toString() + "; it is currently set to [" + msUri + "]"); } + + client = callEmbeddedMetastore(this.conf); + // instantiate the metastore server handler directly instead of connecting // through the network - client = HiveMetaStore.newRetryingHMSHandler("hive client", this.conf, true); isConnected = true; snapshotActiveConf(); return; @@ -224,6 +234,45 @@ public Void run() throws Exception { open(); } + /** + * Instantiate the metastore server handler directly instead of connecting + * through the network + * + * @param conf Configuration object passed to embedded metastore + * @return embedded client instance + * @throws MetaException + */ + static ThriftHiveMetastore.Iface callEmbeddedMetastore(Configuration conf) throws MetaException { + // Instantiate the metastore server handler directly instead of connecting + // through the network + // + // The code below simulates the following code + // + // client = HiveMetaStore.newRetryingHMSHandler(this.conf); + // + // using reflection API. This is done to avoid dependency of MetastoreClient on Hive Metastore. + // Note that newRetryingHMSHandler is static method, so we pass null as the object reference. + // + try { + Class clazz = Class.forName(HIVE_METASTORE_CLASS); + //noinspection JavaReflectionMemberAccess + Method method = clazz.getDeclaredMethod(HIVE_METASTORE_CREATE_HANDLER_METHOD, + Configuration.class); + method.setAccessible(true); + return (ThriftHiveMetastore.Iface) method.invoke(null, conf); + } catch (InvocationTargetException e) { + if (e.getCause() != null) { + MetaStoreUtils.logAndThrowMetaException((Exception)e.getCause()); + } + MetaStoreUtils.logAndThrowMetaException(e); + } catch (ClassNotFoundException + | NoSuchMethodException + | IllegalAccessException e) { + MetaStoreUtils.logAndThrowMetaException(e); + } + return null; + } + private void resolveUris() throws MetaException { String metastoreUrisString[] = MetastoreConf.getVar(conf, ConfVars.THRIFT_URIS).split(","); diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java index 34055d2d4d..35abd006d4 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java @@ -18,9 +18,9 @@ package org.apache.hadoop.hive.metastore; +import static org.apache.hadoop.hive.metastore.HiveMetaStoreClient.callEmbeddedMetastore; import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_DATABASE_NAME; import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCatalog; -import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.prependCatalogToDbName; import java.io.IOException; import java.lang.reflect.Constructor; @@ -162,9 +162,9 @@ public HiveMetaStoreClientPreCatalog(Configuration conf, HiveMetaHookLoader hook throw new MetaException("Embedded metastore is not allowed here. Please configure " + ConfVars.THRIFT_URIS.toString() + "; it is currently set to [" + msUri + "]"); } - // instantiate the metastore server handler directly instead of connecting - // through the network - client = HiveMetaStore.newRetryingHMSHandler("hive client", this.conf, true); + + client = callEmbeddedMetastore(this.conf); + isConnected = true; snapshotActiveConf(); return; -- 2.16.3