From e4fb343d86027ede227da6613e98f16c0a21c08e Mon Sep 17 00:00:00 2001 From: Ashutosh Chauhan Date: Sat, 25 Apr 2020 23:28:23 -0700 Subject: [PATCH] HIVE-23294 : Remove sync bottleneck in TezConfigurationFactory --- .../ql/exec/tez/TezConfigurationFactory.java | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezConfigurationFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezConfigurationFactory.java index a0da0ad2fe..84ae54157e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezConfigurationFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezConfigurationFactory.java @@ -19,6 +19,7 @@ */ package org.apache.hadoop.hive.ql.exec.tez; +import java.lang.reflect.Field; import java.util.Iterator; import java.util.Map; import java.util.function.Predicate; @@ -33,6 +34,7 @@ public class TezConfigurationFactory { private static TezConfiguration defaultConf = new TezConfiguration(); + private static final Field updatingResource; private static final Logger LOG = LoggerFactory.getLogger(TezConfigurationFactory.class.getName()); @@ -41,6 +43,14 @@ String sslConf = defaultConf.get(SSL_CLIENT_CONF_KEY, "ssl-client.xml"); defaultConf.addResource(sslConf); LOG.info("SSL conf : " + sslConf); + try { + //Cache the field handle so that we can avoid expensive conf.getPropertySources(key) later + updatingResource = Configuration.class.getDeclaredField("updatingResource"); + } catch (NoSuchFieldException | SecurityException e) { + throw new RuntimeException(e); + } + updatingResource.setAccessible(true); + } public static Configuration copyInto(Configuration target, Configuration src, @@ -50,7 +60,12 @@ public static Configuration copyInto(Configuration target, Configuration src, Map.Entry entry = iter.next(); String name = entry.getKey(); String value = entry.getValue(); - String[] sources = src.getPropertySources(name); + String[] sources; + try { + sources = ((Map)updatingResource.get(src)).get(name); + } catch (IllegalArgumentException | IllegalAccessException e) { + throw new RuntimeException(e); + } final String source; if (sources == null || sources.length == 0) { source = null; -- 2.17.2 (Apple Git-113)