diff --git itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index 7b68a21..e2fce32 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -73,6 +73,7 @@ import org.apache.hadoop.hive.common.io.SortPrintStream; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.llap.io.api.LlapIoProxy; import org.apache.hadoop.hive.metastore.MetaStoreUtils; import org.apache.hadoop.hive.metastore.api.Index; import org.apache.hadoop.hive.ql.exec.FunctionRegistry; @@ -80,6 +81,7 @@ import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.exec.spark.session.SparkSession; import org.apache.hadoop.hive.ql.exec.spark.session.SparkSessionManagerImpl; +import org.apache.hadoop.hive.llap.io.api.LlapIoProxy; import org.apache.hadoop.hive.ql.lockmgr.zookeeper.CuratorFrameworkSingleton; import org.apache.hadoop.hive.ql.lockmgr.zookeeper.ZooKeeperHiveLockManager; import org.apache.hadoop.hive.ql.metadata.Hive; @@ -411,6 +413,9 @@ public QTestUtil(String outDir, String logDir, MiniClusterType clusterType, initConf(); + LOG.info("initializing llap"); + LlapIoProxy.initializeLlapIo(conf); + // Use the current directory if it is not specified String dataDir = conf.get("test.data.files"); if (dataDir == null) { diff --git llap-client/src/java/org/apache/hadoop/hive/llap/io/api/LlapIoProxy.java llap-client/src/java/org/apache/hadoop/hive/llap/io/api/LlapIoProxy.java index 34df11c..5d3daf0 100644 --- llap-client/src/java/org/apache/hadoop/hive/llap/io/api/LlapIoProxy.java +++ llap-client/src/java/org/apache/hadoop/hive/llap/io/api/LlapIoProxy.java @@ -29,23 +29,18 @@ // Llap server depends on Hive execution, so the reverse cannot be true. We create the I/O // singleton once (on daemon startup); the said singleton server as the IO interface. - private static final Object instanceLock = new Object(); private static LlapIo io = null; - - // TODO: temporary interface. LLAP daemon should init this once during - // startup then others should use get w/o configuration. - public static LlapIo getOrCreateIo(Configuration conf) { - if (io != null) return io; - synchronized (instanceLock) { - if (io != null) return io; - initializeLlapIo(conf); - return io; - } + public static LlapIo getIo() { + return io; } public static void initializeLlapIo(Configuration conf) { - assert io == null; + + if (io != null) { + return; // already initialized + } + try { io = createIoImpl(conf); } catch (IOException e) { @@ -53,10 +48,6 @@ public static void initializeLlapIo(Configuration conf) { } } - public static LlapIo getIo() { - return io; - } - private static LlapIo createIoImpl(Configuration conf) throws IOException { try { @SuppressWarnings("unchecked") diff --git llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java index 1a66568..0b38122 100644 --- llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java +++ llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java @@ -26,6 +26,7 @@ import org.apache.hadoop.hive.llap.daemon.ContainerRunner; import org.apache.hadoop.hive.llap.daemon.LlapDaemonConfiguration; import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.RunContainerRequestProto; +import org.apache.hadoop.hive.llap.io.api.LlapIoProxy; import org.apache.hadoop.hive.llap.shufflehandler.ShuffleHandler; public class LlapDaemon extends AbstractService implements ContainerRunner { @@ -84,6 +85,7 @@ public LlapDaemon(LlapDaemonConfiguration daemonConf) { public void serviceInit(Configuration conf) { server.init(conf); containerRunner.init(conf); + LlapIoProxy.initializeLlapIo(conf); } @Override diff --git ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java index ea8cc43..831d317 100755 --- ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java @@ -83,7 +83,7 @@ /** * A cache of InputFormat instances. */ - private static Map> inputFormats + private static Map> inputFormats = new ConcurrentHashMap>(); private JobConf job; @@ -213,8 +213,7 @@ public void configure(JobConf job) { } LOG.info("Wrapping " + inputFormat); @SuppressWarnings("unchecked") - // TODO: should be LlapIoProxy.getIo eventually - LlapIo llapIo = LlapIoProxy.getOrCreateIo(conf); + LlapIo llapIo = LlapIoProxy.getIo(); return castInputFormat(llapIo.getInputFormat(inputFormat)); }