From 79a669c9ed31e38f857afdacd4ae84cd0b96efb6 Mon Sep 17 00:00:00 2001 From: Alexander Kolbasov Date: Fri, 20 Jul 2018 11:15:04 -0700 Subject: [PATCH 1/1] HIVE-20194: HiveMetastoreClient should use reflection to instantiate embedded HMS instance --- .../hadoop/hive/metastore/HiveMetaStore.java | 18 ++++++++-- .../hadoop/hive/metastore/HiveMetaStoreClient.java | 40 +++++++++++++++++++++- .../metastore/HiveMetaStoreClientPreCatalog.java | 35 ++++++++++++++++++- 3 files changed, 88 insertions(+), 5 deletions(-) diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 47f819b60a..8bff90fc60 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -8729,10 +8729,22 @@ private static IHMSHandler newRetryingHMSHandler(IHMSHandler baseHandler, Config return RetryingHMSHandler.getProxy(conf, baseHandler, local); } - static Iface newRetryingHMSHandler(String name, Configuration conf, boolean local) + /** + * Create retrying HMS handler for embedded metastore. + * + *

IMPORTANT

+ * + * This method is called indirectly by HiveMetastoreClient and HiveMetaStoreClientPreCatalog + * using reflection. It can not be removed and its arguments can't be changed without matching + * change in HiveMetastoreClient and HiveMetaStoreClientPreCatalog. + * + * @param conf configuration to use + * @throws MetaException + */ + static Iface newRetryingHMSHandler(Configuration conf) throws MetaException { - HMSHandler baseHandler = new HiveMetaStore.HMSHandler(name, conf, false); - return RetryingHMSHandler.getProxy(conf, baseHandler, local); + HMSHandler baseHandler = new HiveMetaStore.HMSHandler("hive client", conf, false); + return RetryingHMSHandler.getProxy(conf, baseHandler, true); } /** diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java index 92e280570f..f0e5b52e81 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java @@ -107,6 +107,14 @@ public final static ClientCapabilities TEST_VERSION = new ClientCapabilities( Lists.newArrayList(ClientCapability.INSERT_ONLY_TABLES, ClientCapability.TEST_CAPABILITY)); + // Name of the HiveMetaStore class. It is used to initialize embedded metastore + private static final String HIVE_METASTORE_CLASS = + "org.apache.hadoop.hive.metastore.HiveMetaStore"; + + // Method used to create Hive Metastore client. It is called as + // HiveMetaStore.newRetryingHMSHandler("hive client", this.conf, true); + private static final String HIVE_METASTORE_CREATE_HANDLER_METHOD = "newRetryingHMSHandler"; + ThriftHiveMetastore.Iface client = null; private TTransport transport = null; private boolean isConnected = false; @@ -164,9 +172,39 @@ public HiveMetaStoreClient(Configuration conf, HiveMetaHookLoader hookLoader, Bo throw new MetaException("Embedded metastore is not allowed here. Please configure " + ConfVars.THRIFT_URIS.toString() + "; it is currently set to [" + msUri + "]"); } + + // Instantiate the metastore server handler directly instead of connecting + // through the network + // + // The code below simulates the following code + // + // client = HiveMetaStore.newRetryingHMSHandler(this.conf); + // + // using reflection API. This is done to avoid dependency of MetastoreClient on Hive Metastore. + // Note that newRetryingHMSHandler is static method, so we pass null as the object reference. + // + try { + Class clazz = Class.forName(HIVE_METASTORE_CLASS); + //noinspection JavaReflectionMemberAccess + Method method = clazz.getDeclaredMethod(HIVE_METASTORE_CREATE_HANDLER_METHOD, + Configuration.class); + method.setAccessible(true); + + client = + (ThriftHiveMetastore.Iface) method.invoke(null, this.conf); + } catch (InvocationTargetException e) { + if (e.getCause() != null) { + MetaStoreUtils.logAndThrowMetaException((Exception)e.getCause()); + } + MetaStoreUtils.logAndThrowMetaException(e); + } catch (ClassNotFoundException + | NoSuchMethodException + | IllegalAccessException e) { + MetaStoreUtils.logAndThrowMetaException(e); + } + // instantiate the metastore server handler directly instead of connecting // through the network - client = HiveMetaStore.newRetryingHMSHandler("hive client", this.conf, true); isConnected = true; snapshotActiveConf(); return; diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java index d91f737450..47c8b05e71 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java @@ -106,6 +106,14 @@ public final static ClientCapabilities TEST_VERSION = new ClientCapabilities( Lists.newArrayList(ClientCapability.INSERT_ONLY_TABLES, ClientCapability.TEST_CAPABILITY)); + // Name of the HiveMetaStore class. It is used to initialize embedded metastore + private static final String HIVE_METASTORE_CLASS = + "org.apache.hadoop.hive.metastore.HiveMetaStore"; + + // Method used to create Hive Metastore client. It is called as + // HiveMetaStore.newRetryingHMSHandler("hive client", this.conf, true); + private static final String HIVE_METASTORE_CREATE_HANDLER_METHOD = "newRetryingHMSHandler"; + ThriftHiveMetastore.Iface client = null; private TTransport transport = null; private boolean isConnected = false; @@ -161,9 +169,34 @@ public HiveMetaStoreClientPreCatalog(Configuration conf, HiveMetaHookLoader hook throw new MetaException("Embedded metastore is not allowed here. Please configure " + ConfVars.THRIFT_URIS.toString() + "; it is currently set to [" + msUri + "]"); } + + // // instantiate the metastore server handler directly instead of connecting // through the network - client = HiveMetaStore.newRetryingHMSHandler("hive client", this.conf, true); + // + // The code below simulates the following code + // + // client = HiveMetaStore.newRetryingHMSHandler("hive client", this.conf, true); + // + // using reflection API. This is done to avoid dependency of MetastoreClient on Hive Metastore. + // Note that newRetryingHMSHandler is static method, so we pass null as the object reference. + // + try { + Class clazz = Class.forName(HIVE_METASTORE_CLASS); + //noinspection JavaReflectionMemberAccess + Method method = clazz.getDeclaredMethod(HIVE_METASTORE_CREATE_HANDLER_METHOD, + String.class, Configuration.class, Boolean.TYPE); + method.setAccessible(true); + + client = + (ThriftHiveMetastore.Iface)method.invoke(null, this.conf); + } catch (ClassNotFoundException + | NoSuchMethodException + | InvocationTargetException + | IllegalAccessException e) { + MetaStoreUtils.logAndThrowMetaException(e); + } + isConnected = true; snapshotActiveConf(); return; -- 2.16.3