diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 4676e15..d02c5da 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -562,6 +562,7 @@ public void shutdown() { LOG.debug("RawStore: " + this + ", with PersistenceManager: " + pm + " will be shutdown"); pm.close(); + pm = null; } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index ceb8079..0a5dadc 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -74,10 +74,12 @@ import org.apache.hadoop.hive.metastore.HiveMetaException; import org.apache.hadoop.hive.metastore.HiveMetaHook; import org.apache.hadoop.hive.metastore.HiveMetaHookLoader; +import org.apache.hadoop.hive.metastore.HiveMetaStore; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.metastore.MetaStoreUtils; import org.apache.hadoop.hive.metastore.PartitionDropOptions; +import org.apache.hadoop.hive.metastore.RawStore; import org.apache.hadoop.hive.metastore.RetryingMetaStoreClient; import org.apache.hadoop.hive.metastore.SynchronizedMetaStoreClient; import org.apache.hadoop.hive.metastore.TableType; @@ -412,6 +414,9 @@ private void close() { metaStoreClient.close(); metaStoreClient = null; } + if (syncMetaStoreClient != null) { + syncMetaStoreClient.close(); + } if (owner != null) { owner = null; } @@ -1971,6 +1976,7 @@ private void constructOneLBLocationMap(FileStatus fSta, try { // for each dynamically created DP directory, construct a full partition spec // and load the partition based on that + final Map rawStoreMap = new HashMap(); for(final Path partPath : validPartitions) { // generate a full partition specification final LinkedHashMap fullPartSpec = Maps.newLinkedHashMap(partSpec); @@ -1997,6 +2003,12 @@ public Void call() throws Exception { + partsToLoad + " partitions."); } } + // Add embedded rawstore, so we can cleanup later to avoid memory leak + if (getMSC().isLocalMetaStore()) { + if (!rawStoreMap.containsKey(Thread.currentThread().getId())) { + rawStoreMap.put(Thread.currentThread().getId(), HiveMetaStore.HMSHandler.getRawStore()); + } + } return null; } catch (Exception t) { LOG.error("Exception when loading partition with parameters " @@ -2018,6 +2030,10 @@ public Void call() throws Exception { for (Future future : futures) { future.get(); } + + for (RawStore rs : rawStoreMap.values()) { + rs.shutdown(); + } } catch (InterruptedException | ExecutionException e) { LOG.debug("Cancelling " + futures.size() + " dynamic loading tasks"); //cancel other futures