diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezConfigurationFactory.java ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezConfigurationFactory.java index a0da0ad2fe..4ea81758d1 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezConfigurationFactory.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezConfigurationFactory.java @@ -21,6 +21,7 @@ import java.util.Iterator; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import java.util.function.Predicate; import org.apache.hadoop.conf.Configuration; @@ -33,6 +34,7 @@ public class TezConfigurationFactory { private static TezConfiguration defaultConf = new TezConfiguration(); + private static Map sourceMap = new ConcurrentHashMap<>(); private static final Logger LOG = LoggerFactory.getLogger(TezConfigurationFactory.class.getName()); @@ -50,7 +52,8 @@ public static Configuration copyInto(Configuration target, Configuration src, Map.Entry entry = iter.next(); String name = entry.getKey(); String value = entry.getValue(); - String[] sources = src.getPropertySources(name); + //TODO: since it is property sources, it should be fine. + String[] sources = sourceMap.computeIfAbsent(name, v -> src.getPropertySources(name)); final String source; if (sources == null || sources.length == 0) { source = null; @@ -67,6 +70,11 @@ public static Configuration copyInto(Configuration target, Configuration src, } else { } } + //TODO: temp capacity checks + if (sourceMap.size() > 10000) { + LOG.info("Clearing source map details"); + sourceMap.clear(); + } return target; }