From 6e1961555a5e137801e735044c210a189c7aaa37 Mon Sep 17 00:00:00 2001 From: Alexander Kolbasov Date: Wed, 7 Mar 2018 17:16:40 -0800 Subject: [PATCH 1/1] HIVE-18888: Replace synchronizedMap with ConcurrentHashMap --- .../hive/ql/exec/tez/DynamicValueRegistryTez.java | 19 +++++++------------ .../java/org/apache/hadoop/hive/ql/metadata/Hive.java | 14 ++------------ .../apache/hadoop/hive/metastore/HiveMetaStore.java | 7 +++---- 3 files changed, 12 insertions(+), 28 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DynamicValueRegistryTez.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DynamicValueRegistryTez.java index 0bed22a8f8..ec1e84b10f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DynamicValueRegistryTez.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DynamicValueRegistryTez.java @@ -18,35 +18,30 @@ package org.apache.hadoop.hive.ql.exec.tez; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.ql.exec.DynamicValueRegistry; import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator; import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluatorFactory; -import org.apache.hadoop.hive.ql.exec.DynamicValueRegistry; -import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.parse.RuntimeValuesInfo; import org.apache.hadoop.hive.ql.plan.BaseWork; import org.apache.hadoop.hive.ql.plan.DynamicValue.NoDynamicValuesException; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; -import org.apache.hadoop.hive.ql.plan.TableDesc; import org.apache.hadoop.hive.serde2.Deserializer; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.io.Writable; import org.apache.hadoop.util.ReflectionUtils; import org.apache.tez.runtime.api.Input; import org.apache.tez.runtime.api.LogicalInput; import org.apache.tez.runtime.api.ProcessorContext; import org.apache.tez.runtime.library.api.KeyValueReader; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + public class DynamicValueRegistryTez implements DynamicValueRegistry { private static final Logger LOG = LoggerFactory.getLogger(DynamicValueRegistryTez.class); @@ -66,7 +61,7 @@ public RegistryConfTez(Configuration conf, BaseWork baseWork, } } - protected Map values = Collections.synchronizedMap(new HashMap()); + protected Map values = new ConcurrentHashMap<>(); public DynamicValueRegistryTez() { } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index c0be51e0b2..c5c71c4b48 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -37,7 +37,6 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.LinkedList; @@ -174,13 +173,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.base.Splitter; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; -import com.google.common.util.concurrent.ThreadFactoryBuilder; - /** * This class has functions that implement meta data/DDL operations using calls * to the metastore. @@ -2042,7 +2034,7 @@ private void constructOneLBLocationMap(FileStatus fSta, try { // for each dynamically created DP directory, construct a full partition spec // and load the partition based on that - final Map rawStoreMap = Collections.synchronizedMap(new HashMap()); + final Map rawStoreMap = new ConcurrentHashMap<>(); for(final Path partPath : validPartitions) { // generate a full partition specification final LinkedHashMap fullPartSpec = Maps.newLinkedHashMap(partSpec); @@ -2101,9 +2093,7 @@ public Void call() throws Exception { future.get(); } - for (RawStore rs : rawStoreMap.values()) { - rs.shutdown(); - } + rawStoreMap.forEach((k, rs) -> rs.shutdown()); } catch (InterruptedException | ExecutionException e) { LOG.debug("Cancelling " + futures.size() + " dynamic loading tasks"); //cancel other futures diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 662de9a667..16d2cb42e2 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -43,6 +43,7 @@ import java.util.Properties; import java.util.Set; import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -2852,8 +2853,7 @@ public boolean equals(Object obj) { logInfo("add_partitions"); boolean success = false; // Ensures that the list doesn't have dups, and keeps track of directories we have created. - final Map addedPartitions = - Collections.synchronizedMap(new HashMap()); + final Map addedPartitions = new ConcurrentHashMap<>(); final List newParts = new ArrayList<>(); final List existingParts = new ArrayList<>(); Table tbl = null; @@ -3063,8 +3063,7 @@ private int add_partitions_pspec_core( throws TException { boolean success = false; // Ensures that the list doesn't have dups, and keeps track of directories we have created. - final Map addedPartitions = - Collections.synchronizedMap(new HashMap()); + final Map addedPartitions = new ConcurrentHashMap<>(); PartitionSpecProxy partitionSpecProxy = PartitionSpecProxy.Factory.get(partSpecs); final PartitionSpecProxy.PartitionIterator partitionIterator = partitionSpecProxy .getPartitionIterator(); -- 2.16.2