From b6c7cb06af59e3a4dc95a1bb84a2bd601965a5e7 Mon Sep 17 00:00:00 2001 From: Alexander Kolbasov Date: Tue, 6 Mar 2018 22:06:00 -0800 Subject: [PATCH 1/1] HIVE-18888: Replace synchronizedMap with ConcurrentHashMap --- .../apache/hadoop/hive/ql/exec/tez/DynamicValueRegistryTez.java | 3 ++- ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java | 6 ++---- .../main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java | 7 +++---- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DynamicValueRegistryTez.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DynamicValueRegistryTez.java index 0bed22a8f8..02835a845a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DynamicValueRegistryTez.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DynamicValueRegistryTez.java @@ -23,6 +23,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator; @@ -66,7 +67,7 @@ public RegistryConfTez(Configuration conf, BaseWork baseWork, } } - protected Map values = Collections.synchronizedMap(new HashMap()); + protected Map values = new ConcurrentHashMap<>(); public DynamicValueRegistryTez() { } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index c0be51e0b2..b1323c9dd1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -2042,7 +2042,7 @@ private void constructOneLBLocationMap(FileStatus fSta, try { // for each dynamically created DP directory, construct a full partition spec // and load the partition based on that - final Map rawStoreMap = Collections.synchronizedMap(new HashMap()); + final Map rawStoreMap = new ConcurrentHashMap<>(); for(final Path partPath : validPartitions) { // generate a full partition specification final LinkedHashMap fullPartSpec = Maps.newLinkedHashMap(partSpec); @@ -2101,9 +2101,7 @@ public Void call() throws Exception { future.get(); } - for (RawStore rs : rawStoreMap.values()) { - rs.shutdown(); - } + rawStoreMap.forEach((k, rs) -> rs.shutdown()); } catch (InterruptedException | ExecutionException e) { LOG.debug("Cancelling " + futures.size() + " dynamic loading tasks"); //cancel other futures diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 2be018ba0f..2b58ec2a41 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -43,6 +43,7 @@ import java.util.Properties; import java.util.Set; import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -2865,8 +2866,7 @@ public boolean equals(Object obj) { logInfo("add_partitions"); boolean success = false; // Ensures that the list doesn't have dups, and keeps track of directories we have created. - final Map addedPartitions = - Collections.synchronizedMap(new HashMap()); + final Map addedPartitions = new ConcurrentHashMap<>(); final List newParts = new ArrayList<>(); final List existingParts = new ArrayList<>(); Table tbl = null; @@ -3076,8 +3076,7 @@ private int add_partitions_pspec_core( throws TException { boolean success = false; // Ensures that the list doesn't have dups, and keeps track of directories we have created. - final Map addedPartitions = - Collections.synchronizedMap(new HashMap()); + final Map addedPartitions = new ConcurrentHashMap<>(); PartitionSpecProxy partitionSpecProxy = PartitionSpecProxy.Factory.get(partSpecs); final PartitionSpecProxy.PartitionIterator partitionIterator = partitionSpecProxy .getPartitionIterator(); -- 2.16.2