From 9705a3ed7b298f9c5e1175d87f3d4414f9158c3f Mon Sep 17 00:00:00 2001 From: Alexander Kolbasov Date: Tue, 17 Jul 2018 17:07:38 -0700 Subject: [PATCH 1/1] HIVE-20194: HiveMetastoreClient should use reflection to instantiate embedded HMS instance --- .../hadoop/hive/metastore/HiveMetaStore.java | 14 ++++++++ .../hadoop/hive/metastore/HiveMetaStoreClient.java | 37 +++++++++++++++++++++- .../metastore/HiveMetaStoreClientPreCatalog.java | 35 +++++++++++++++++++- 3 files changed, 84 insertions(+), 2 deletions(-) diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 47f819b60a..3e4428fa4a 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -8729,6 +8729,20 @@ private static IHMSHandler newRetryingHMSHandler(IHMSHandler baseHandler, Config return RetryingHMSHandler.getProxy(conf, baseHandler, local); } + /** + * Create retrying HMS handler for embedded metastore. + * + *

IMPORTANT

+ * + * This method is called indirectly by HiveMetastoreClient and HiveMetaStoreClientPreCatalog + * using reflection. It can not be removed and its arguments can't be changed without matching + * change in HiveMetastoreClient and HiveMetaStoreClientPreCatalog. + * + * @param name HMS name + * @param conf configuration to use + * @param local whether it is using local or remote metastore + * @throws MetaException + */ static Iface newRetryingHMSHandler(String name, Configuration conf, boolean local) throws MetaException { HMSHandler baseHandler = new HiveMetaStore.HMSHandler(name, conf, false); diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java index 92e280570f..56abe2c6b1 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java @@ -107,6 +107,14 @@ public final static ClientCapabilities TEST_VERSION = new ClientCapabilities( Lists.newArrayList(ClientCapability.INSERT_ONLY_TABLES, ClientCapability.TEST_CAPABILITY)); + // Name of the HiveMetaStore class. It is used to initialize embedded metastore + private static final String HIVE_METASTORE_CLASS = + "org.apache.hadoop.hive.metastore.HiveMetaStore"; + + // Method used to create Hive Metastore client. It is called as + // HiveMetaStore.newRetryingHMSHandler("hive client", this.conf, true); + private static final String HIVE_METASTORE_CREATE_HANDLER_METHOD = "newRetryingHMSHandler"; + ThriftHiveMetastore.Iface client = null; private TTransport transport = null; private boolean isConnected = false; @@ -164,9 +172,36 @@ public HiveMetaStoreClient(Configuration conf, HiveMetaHookLoader hookLoader, Bo throw new MetaException("Embedded metastore is not allowed here. Please configure " + ConfVars.THRIFT_URIS.toString() + "; it is currently set to [" + msUri + "]"); } + + // + // instantiate the metastore server handler directly instead of connecting + // through the network + // + // The code below simulates the following code + // + // client = HiveMetaStore.newRetryingHMSHandler("hive client", this.conf, true); + // + // using reflection API. This is done to avoid dependency of MetastoreClient on Hive Metastore. + // Note that newRetryingHMSHandler is static method, so we pass null as the object reference. + // + try { + Class clazz = Class.forName(HIVE_METASTORE_CLASS); + //noinspection JavaReflectionMemberAccess + Method method = clazz.getDeclaredMethod(HIVE_METASTORE_CREATE_HANDLER_METHOD, + String.class, Configuration.class, Boolean.TYPE); + method.setAccessible(true); + + client = + (ThriftHiveMetastore.Iface)method.invoke(null, "hive client", this.conf, true); + } catch (ClassNotFoundException + | NoSuchMethodException + | InvocationTargetException + | IllegalAccessException e) { + MetaStoreUtils.logAndThrowMetaException(e); + } + // instantiate the metastore server handler directly instead of connecting // through the network - client = HiveMetaStore.newRetryingHMSHandler("hive client", this.conf, true); isConnected = true; snapshotActiveConf(); return; diff --git a/standalone-metastore/metastore-common/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java b/standalone-metastore/metastore-common/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java index d91f737450..f0b1d77949 100644 --- a/standalone-metastore/metastore-common/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java +++ b/standalone-metastore/metastore-common/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java @@ -106,6 +106,14 @@ public final static ClientCapabilities TEST_VERSION = new ClientCapabilities( Lists.newArrayList(ClientCapability.INSERT_ONLY_TABLES, ClientCapability.TEST_CAPABILITY)); + // Name of the HiveMetaStore class. It is used to initialize embedded metastore + private static final String HIVE_METASTORE_CLASS = + "org.apache.hadoop.hive.metastore.HiveMetaStore"; + + // Method used to create Hive Metastore client. It is called as + // HiveMetaStore.newRetryingHMSHandler("hive client", this.conf, true); + private static final String HIVE_METASTORE_CREATE_HANDLER_METHOD = "newRetryingHMSHandler"; + ThriftHiveMetastore.Iface client = null; private TTransport transport = null; private boolean isConnected = false; @@ -161,9 +169,34 @@ public HiveMetaStoreClientPreCatalog(Configuration conf, HiveMetaHookLoader hook throw new MetaException("Embedded metastore is not allowed here. Please configure " + ConfVars.THRIFT_URIS.toString() + "; it is currently set to [" + msUri + "]"); } + + // // instantiate the metastore server handler directly instead of connecting // through the network - client = HiveMetaStore.newRetryingHMSHandler("hive client", this.conf, true); + // + // The code below simulates the following code + // + // client = HiveMetaStore.newRetryingHMSHandler("hive client", this.conf, true); + // + // using reflection API. This is done to avoid dependency of MetastoreClient on Hive Metastore. + // Note that newRetryingHMSHandler is static method, so we pass null as the object reference. + // + try { + Class clazz = Class.forName(HIVE_METASTORE_CLASS); + //noinspection JavaReflectionMemberAccess + Method method = clazz.getDeclaredMethod(HIVE_METASTORE_CREATE_HANDLER_METHOD, + String.class, Configuration.class, Boolean.TYPE); + method.setAccessible(true); + + client = + (ThriftHiveMetastore.Iface)method.invoke(null, "hive client", this.conf, true); + } catch (ClassNotFoundException + | NoSuchMethodException + | InvocationTargetException + | IllegalAccessException e) { + MetaStoreUtils.logAndThrowMetaException(e); + } + isConnected = true; snapshotActiveConf(); return; -- 2.16.3