From bc2bf0a64afc0865325424ec2f6a6514974d0e68 Mon Sep 17 00:00:00 2001 From: Alexander Kolbasov Date: Thu, 9 Aug 2018 15:33:45 -0700 Subject: [PATCH 1/1] HIVE-20194: HiveMetastoreClient should use reflection to instantiate embedded HMS instance --- .../hadoop/hive/metastore/HiveMetaStore.java | 18 ++++++++-- .../hadoop/hive/metastore/HiveMetaStoreClient.java | 40 +++++++++++++++++++++- .../metastore/HiveMetaStoreClientPreCatalog.java | 39 ++++++++++++++++++++- 3 files changed, 92 insertions(+), 5 deletions(-) diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index a53d4be03d..d34e94414f 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -8979,10 +8979,22 @@ private static IHMSHandler newRetryingHMSHandler(IHMSHandler baseHandler, Config return RetryingHMSHandler.getProxy(conf, baseHandler, local); } - static Iface newRetryingHMSHandler(String name, Configuration conf, boolean local) + /** + * Create retrying HMS handler for embedded metastore. + * + *

IMPORTANT

+ * + * This method is called indirectly by HiveMetastoreClient and HiveMetaStoreClientPreCatalog + * using reflection. It can not be removed and its arguments can't be changed without matching + * change in HiveMetastoreClient and HiveMetaStoreClientPreCatalog. + * + * @param conf configuration to use + * @throws MetaException + */ + static Iface newRetryingHMSHandler(Configuration conf) throws MetaException { - HMSHandler baseHandler = new HiveMetaStore.HMSHandler(name, conf, false); - return RetryingHMSHandler.getProxy(conf, baseHandler, local); + HMSHandler baseHandler = new HiveMetaStore.HMSHandler("hive client", conf, false); + return RetryingHMSHandler.getProxy(conf, baseHandler, true); } /** diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java index 91c86a749c..8037a73588 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java @@ -107,6 +107,14 @@ public final static ClientCapabilities TEST_VERSION = new ClientCapabilities( Lists.newArrayList(ClientCapability.INSERT_ONLY_TABLES, ClientCapability.TEST_CAPABILITY)); + // Name of the HiveMetaStore class. It is used to initialize embedded metastore + private static final String HIVE_METASTORE_CLASS = + "org.apache.hadoop.hive.metastore.HiveMetaStore"; + + // Method used to create Hive Metastore client. It is called as + // HiveMetaStore.newRetryingHMSHandler("hive client", this.conf, true); + private static final String HIVE_METASTORE_CREATE_HANDLER_METHOD = "newRetryingHMSHandler"; + ThriftHiveMetastore.Iface client = null; private TTransport transport = null; private boolean isConnected = false; @@ -164,9 +172,39 @@ public HiveMetaStoreClient(Configuration conf, HiveMetaHookLoader hookLoader, Bo throw new MetaException("Embedded metastore is not allowed here. Please configure " + ConfVars.THRIFT_URIS.toString() + "; it is currently set to [" + msUri + "]"); } + + // Instantiate the metastore server handler directly instead of connecting + // through the network + // + // The code below simulates the following code + // + // client = HiveMetaStore.newRetryingHMSHandler(this.conf); + // + // using reflection API. This is done to avoid dependency of MetastoreClient on Hive Metastore. + // Note that newRetryingHMSHandler is static method, so we pass null as the object reference. + // + try { + Class clazz = Class.forName(HIVE_METASTORE_CLASS); + //noinspection JavaReflectionMemberAccess + Method method = clazz.getDeclaredMethod(HIVE_METASTORE_CREATE_HANDLER_METHOD, + Configuration.class); + method.setAccessible(true); + + client = + (ThriftHiveMetastore.Iface) method.invoke(null, this.conf); + } catch (InvocationTargetException e) { + if (e.getCause() != null) { + MetaStoreUtils.logAndThrowMetaException((Exception)e.getCause()); + } + MetaStoreUtils.logAndThrowMetaException(e); + } catch (ClassNotFoundException + | NoSuchMethodException + | IllegalAccessException e) { + MetaStoreUtils.logAndThrowMetaException(e); + } + // instantiate the metastore server handler directly instead of connecting // through the network - client = HiveMetaStore.newRetryingHMSHandler("hive client", this.conf, true); isConnected = true; snapshotActiveConf(); return; diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java index 34055d2d4d..a45e73eefc 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java @@ -107,6 +107,14 @@ public final static ClientCapabilities TEST_VERSION = new ClientCapabilities( Lists.newArrayList(ClientCapability.INSERT_ONLY_TABLES, ClientCapability.TEST_CAPABILITY)); + // Name of the HiveMetaStore class. It is used to initialize embedded metastore + private static final String HIVE_METASTORE_CLASS = + "org.apache.hadoop.hive.metastore.HiveMetaStore"; + + // Method used to create Hive Metastore client. It is called as + // HiveMetaStore.newRetryingHMSHandler("hive client", this.conf, true); + private static final String HIVE_METASTORE_CREATE_HANDLER_METHOD = "newRetryingHMSHandler"; + ThriftHiveMetastore.Iface client = null; private TTransport transport = null; private boolean isConnected = false; @@ -162,9 +170,38 @@ public HiveMetaStoreClientPreCatalog(Configuration conf, HiveMetaHookLoader hook throw new MetaException("Embedded metastore is not allowed here. Please configure " + ConfVars.THRIFT_URIS.toString() + "; it is currently set to [" + msUri + "]"); } + + // // instantiate the metastore server handler directly instead of connecting // through the network - client = HiveMetaStore.newRetryingHMSHandler("hive client", this.conf, true); + // + // The code below simulates the following code + // + // client = HiveMetaStore.newRetryingHMSHandler(this.conf); + // + // using reflection API. This is done to avoid dependency of MetastoreClient on Hive Metastore. + // Note that newRetryingHMSHandler is static method, so we pass null as the object reference. + // + try { + Class clazz = Class.forName(HIVE_METASTORE_CLASS); + //noinspection JavaReflectionMemberAccess + Method method = clazz.getDeclaredMethod(HIVE_METASTORE_CREATE_HANDLER_METHOD, + Configuration.class); + method.setAccessible(true); + + client = + (ThriftHiveMetastore.Iface)method.invoke(null, this.conf); + } catch (InvocationTargetException e) { + if (e.getCause() != null) { + MetaStoreUtils.logAndThrowMetaException((Exception)e.getCause()); + } + MetaStoreUtils.logAndThrowMetaException(e); + } catch (ClassNotFoundException + | NoSuchMethodException + | IllegalAccessException e) { + MetaStoreUtils.logAndThrowMetaException(e); + } + isConnected = true; snapshotActiveConf(); return; -- 2.16.3