diff --git data/conf/llap/hive-site.xml data/conf/llap/hive-site.xml
index c2bef58..72bdcfb 100644
--- data/conf/llap/hive-site.xml
+++ data/conf/llap/hive-site.xml
@@ -273,4 +273,48 @@
false
+
+
+
+ hive.llap.daemon.service.hosts
+ localhost
+
+
+
+ hive.llap.daemon.service.port
+ 0
+
+
+
+ hive.llap.daemon.num.executors
+ 4
+
+
+
+ hive.llap.daemon.task.scheduler.wait.queue.size
+ 4
+
+
+
+ hive.llap.cache.allow.synthetic.fileid
+ true
+
+
+
+
+ ipc.client.low-latency
+ true
+
+
+
+ ipc.client.tcpnodelay
+ true
+
+
+
+ ipc.clients-per-factory
+ 4
+
+
+
diff --git data/conf/llap/llap-daemon-site.xml data/conf/llap/llap-daemon-site.xml
deleted file mode 100644
index 98c0f2b..0000000
--- data/conf/llap/llap-daemon-site.xml
+++ /dev/null
@@ -1,61 +0,0 @@
-
-
-
-
-
- hive.llap.daemon.service.hosts
- localhost
-
-
-
- hive.llap.daemon.service.port
- 0
-
-
-
- hive.llap.daemon.num.executors
- 4
-
-
-
- hive.llap.daemon.task.scheduler.wait.queue.size
- 4
-
-
-
- hive.llap.cache.allow.synthetic.fileid
- true
-
-
-
-
- ipc.client.low-latency
- true
-
-
-
- ipc.client.tcpnodelay
- true
-
-
-
- ipc.clients-per-factory
- 4
-
-
-
diff --git itests/hive-unit/pom.xml itests/hive-unit/pom.xml
index 97786d9..7219f1d 100644
--- itests/hive-unit/pom.xml
+++ itests/hive-unit/pom.xml
@@ -75,6 +75,11 @@
hive-hcatalog-streaming
${project.version}
+
+ org.apache.hive
+ hive-it-util
+ ${project.version}
+
org.apache.hadoop
@@ -123,12 +128,6 @@
org.apache.hive
- hive-it-util
- ${project.version}
- test
-
-
- org.apache.hive
hive-jdbc
${project.version}
test
diff --git itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java
index 751d8ea..26caa8d 100644
--- itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java
+++ itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java
@@ -31,6 +31,8 @@
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.llap.LlapItUtils;
+import org.apache.hadoop.hive.llap.daemon.MiniLlapCluster;
import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.ql.WindowsPathUtil;
import org.apache.hadoop.hive.ql.exec.Utilities;
@@ -59,6 +61,7 @@
private static final AtomicLong hs2Counter = new AtomicLong();
private MiniMrShim mr;
private MiniDFSShim dfs;
+ private MiniLlapCluster llapCluster = null;
private final FileSystem localFS;
private boolean useMiniKdc = false;
private final String serverPrincipal;
@@ -186,13 +189,15 @@ private MiniHS2(HiveConf hiveConf, MiniClusterType miniClusterType, boolean useM
// Initialize the execution engine based on cluster type
switch (miniClusterType) {
case TEZ:
- mr = ShimLoader.getHadoopShims().getMiniTezCluster(hiveConf, 4, uriString, false);
+ mr = ShimLoader.getHadoopShims().getMiniTezCluster(hiveConf, 4, uriString);
break;
case LLAP:
if (usePortsFromConf) {
hiveConf.setBoolean("minillap.usePortsFromConf", true);
}
- mr = ShimLoader.getHadoopShims().getMiniTezCluster(hiveConf, 4, uriString, true);
+ llapCluster = LlapItUtils.startAndGetMiniLlapCluster(hiveConf, null);
+
+ mr = ShimLoader.getHadoopShims().getMiniTezCluster(hiveConf, 4, uriString);
break;
case MR:
mr = ShimLoader.getHadoopShims().getMiniMrCluster(hiveConf, 4, uriString, 1);
@@ -284,6 +289,9 @@ public void stop() {
hiveServer2.stop();
setStarted(false);
try {
+ if (llapCluster != null) {
+ llapCluster.stop();
+ }
if (mr != null) {
mr.shutdown();
mr = null;
diff --git itests/util/pom.xml itests/util/pom.xml
index aaafc0a..4789586 100644
--- itests/util/pom.xml
+++ itests/util/pom.xml
@@ -97,6 +97,17 @@
tests
+ org.apache.hive
+ hive-llap-server
+ ${project.version}
+
+
+ org.apache.hive
+ hive-llap-server
+ ${project.version}
+ test-jar
+
+
org.apache.hadoop
hadoop-common
${hadoop.version}
diff --git itests/util/src/main/java/org/apache/hadoop/hive/llap/LlapItUtils.java itests/util/src/main/java/org/apache/hadoop/hive/llap/LlapItUtils.java
new file mode 100644
index 0000000..cb4aba5
--- /dev/null
+++ itests/util/src/main/java/org/apache/hadoop/hive/llap/LlapItUtils.java
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.llap;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+import java.util.Iterator;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.llap.configuration.LlapDaemonConfiguration;
+import org.apache.hadoop.hive.llap.daemon.MiniLlapCluster;
+import org.apache.hadoop.hive.llap.daemon.impl.LlapDaemon;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class LlapItUtils {
+
+ private static final Logger LOG = LoggerFactory.getLogger(LlapItUtils.class);
+
+ public static MiniLlapCluster startAndGetMiniLlapCluster(Configuration conf, String confDir) throws
+ IOException {
+ MiniLlapCluster llapCluster;
+ LOG.info("Using conf dir: {}", confDir);
+ if (confDir != null && !confDir.isEmpty()) {
+ conf.addResource(new URL("file://" + new File(confDir).toURI().getPath()
+ + "/tez-site.xml"));
+ }
+
+ Configuration daemonConf = new LlapDaemonConfiguration(conf);
+ final String clusterName = "llap";
+ final long maxMemory = LlapDaemon.getTotalHeapSize();
+ // 15% for io cache
+ final long memoryForCache = (long) (0.15f * maxMemory);
+ // 75% for 4 executors
+ final long totalExecutorMemory = (long) (0.75f * maxMemory);
+ final int numExecutors = HiveConf.getIntVar(conf, HiveConf.ConfVars.LLAP_DAEMON_NUM_EXECUTORS);
+ final boolean asyncIOEnabled = true;
+ // enabling this will cause test failures in Mac OS X
+ final boolean directMemoryEnabled = false;
+ final int numLocalDirs = 1;
+ LOG.info("MiniLlap Configs - maxMemory: " + maxMemory + " memoryForCache: " + memoryForCache
+ + " totalExecutorMemory: " + totalExecutorMemory + " numExecutors: " + numExecutors
+ + " asyncIOEnabled: " + asyncIOEnabled + " directMemoryEnabled: " + directMemoryEnabled
+ + " numLocalDirs: " + numLocalDirs);
+ llapCluster = MiniLlapCluster.create(clusterName,
+ numExecutors,
+ totalExecutorMemory,
+ asyncIOEnabled,
+ directMemoryEnabled,
+ memoryForCache,
+ numLocalDirs);
+ llapCluster.init(daemonConf);
+ llapCluster.start();
+
+ // Augment conf with the settings from the started llap configuration.
+ Configuration llapConf = llapCluster.getClusterSpecificConfiguration();
+ Iterator> confIter = llapConf.iterator();
+ while (confIter.hasNext()) {
+ Map.Entry entry = confIter.next();
+ conf.set(entry.getKey(), entry.getValue());
+ }
+ return llapCluster;
+ }
+
+}
diff --git itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index 5e81e98..5ccbcba 100644
--- itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -51,6 +51,7 @@
import java.util.Comparator;
import java.util.Deque;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -82,6 +83,10 @@
import org.apache.hadoop.hive.common.io.SortPrintStream;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.llap.LlapItUtils;
+import org.apache.hadoop.hive.llap.configuration.LlapDaemonConfiguration;
+import org.apache.hadoop.hive.llap.daemon.MiniLlapCluster;
+import org.apache.hadoop.hive.llap.daemon.impl.LlapDaemon;
import org.apache.hadoop.hive.llap.io.api.LlapProxy;
import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.metastore.api.Index;
@@ -166,6 +171,7 @@
private HadoopShims.MiniMrShim mr = null;
private HadoopShims.MiniDFSShim dfs = null;
private HadoopShims.HdfsEncryptionShim hes = null;
+ private MiniLlapCluster llapCluster = null;
private String hadoopVer = null;
private QTestSetup setup = null;
private TezSessionState tezSessionState = null;
@@ -429,15 +435,10 @@ public QTestUtil(String outDir, String logDir, MiniClusterType clusterType,
conf.addResource(new URL("file://" + new File(confDir).toURI().getPath()
+ "/tez-site.xml"));
}
- mr = shims.getMiniTezCluster(conf, 4, uriString, false);
+ mr = shims.getMiniTezCluster(conf, 4, uriString);
} else if (clusterType == MiniClusterType.llap) {
- if (confDir != null && !confDir.isEmpty()) {
- conf.addResource(new URL("file://" + new File(confDir).toURI().getPath()
- + "/tez-site.xml"));
- conf.addResource(new URL("file://" + new File(confDir).toURI().getPath()
- + "/llap-daemon-site.xml"));
- }
- mr = shims.getMiniTezCluster(conf, 2, uriString, true);
+ llapCluster = LlapItUtils.startAndGetMiniLlapCluster(conf, confDir);
+ mr = shims.getMiniTezCluster(conf, 2, uriString);
} else if (clusterType == MiniClusterType.miniSparkOnYarn) {
mr = shims.getMiniSparkCluster(conf, 4, uriString, 1);
} else {
diff --git llap-server/src/java/org/apache/hadoop/hive/llap/configuration/LlapDaemonConfiguration.java llap-server/src/java/org/apache/hadoop/hive/llap/configuration/LlapDaemonConfiguration.java
index 51e8509..88f3b19 100644
--- llap-server/src/java/org/apache/hadoop/hive/llap/configuration/LlapDaemonConfiguration.java
+++ llap-server/src/java/org/apache/hadoop/hive/llap/configuration/LlapDaemonConfiguration.java
@@ -14,6 +14,7 @@
package org.apache.hadoop.hive.llap.configuration;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.common.classification.InterfaceAudience;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -21,6 +22,7 @@
/**
* Configuration for LLAP daemon processes only. This should not be used by any clients.
*/
+@InterfaceAudience.Private
public class LlapDaemonConfiguration extends Configuration {
@InterfaceAudience.Private
@@ -46,4 +48,10 @@ public LlapDaemonConfiguration() {
}
addResource(LLAP_DAEMON_SITE);
}
+
+ @VisibleForTesting
+ public LlapDaemonConfiguration(Configuration conf) {
+ this();
+ addResource(conf);
+ }
}
\ No newline at end of file
diff --git shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
index 9a3a31c..e028212 100644
--- shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
+++ shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
@@ -370,8 +370,8 @@ public void setupConfiguration(Configuration conf) {
*/
@Override
public MiniMrShim getMiniTezCluster(Configuration conf, int numberOfTaskTrackers,
- String nameNode, boolean isLlap) throws IOException {
- return new MiniTezShim(conf, numberOfTaskTrackers, nameNode, isLlap);
+ String nameNode) throws IOException {
+ return new MiniTezShim(conf, numberOfTaskTrackers, nameNode);
}
/**
@@ -381,11 +381,8 @@ public MiniMrShim getMiniTezCluster(Configuration conf, int numberOfTaskTrackers
private final MiniTezCluster mr;
private final Configuration conf;
- private Class> miniLlapKlass;
- private Object miniLlapCluster;
- public MiniTezShim(Configuration conf, int numberOfTaskTrackers, String nameNode,
- boolean isLlap) throws IOException {
+ public MiniTezShim(Configuration conf, int numberOfTaskTrackers, String nameNode) throws IOException {
mr = new MiniTezCluster("hive", numberOfTaskTrackers);
conf.set("fs.defaultFS", nameNode);
conf.set("tez.am.log.level", "DEBUG");
@@ -393,54 +390,6 @@ public MiniTezShim(Configuration conf, int numberOfTaskTrackers, String nameNode
mr.init(conf);
mr.start();
this.conf = mr.getConfig();
- if (isLlap) {
- createAndLaunchLlapDaemon(this.conf);
- } else {
- miniLlapCluster = null;
- }
- }
-
- private void createAndLaunchLlapDaemon(final Configuration conf)
- throws IOException {
- try {
- final String clusterName = "llap";
- Class> llapDaemonKlass =
- Class.forName("org.apache.hadoop.hive.llap.daemon.impl.LlapDaemon",
- false, ShimLoader.class.getClassLoader());
- Method totalMemMethod = llapDaemonKlass.getMethod("getTotalHeapSize");
- final long maxMemory = (long) totalMemMethod.invoke(null);
- // 15% for io cache
- final long memoryForCache = (long) (0.15f * maxMemory);
- // 75% for executors
- final long totalExecutorMemory = (long) (0.75f * maxMemory);
- final int numExecutors = conf.getInt("llap.daemon.num.executors", 4);
- final boolean asyncIOEnabled = true;
- // enabling this will cause test failures in Mac OS X
- final boolean directMemoryEnabled = false;
- final int numLocalDirs = 1;
- LOG.info("MiniLlap Configs - maxMemory: " + maxMemory + " memoryForCache: " + memoryForCache
- + " totalExecutorMemory: " + totalExecutorMemory + " numExecutors: " + numExecutors
- + " asyncIOEnabled: " + asyncIOEnabled + " directMemoryEnabled: " + directMemoryEnabled
- + " numLocalDirs: " + numLocalDirs);
-
- miniLlapKlass = Class.forName("org.apache.hadoop.hive.llap.daemon.MiniLlapCluster",
- false, ShimLoader.class.getClassLoader());
- Method create = miniLlapKlass.getMethod("createAndLaunch", new Class[]{Configuration.class,
- String.class, Integer.TYPE, Long.TYPE, Boolean.TYPE, Boolean.TYPE,
- Long.TYPE, Integer.TYPE});
- miniLlapCluster = create.invoke(null,
- conf,
- clusterName,
- numExecutors,
- totalExecutorMemory,
- asyncIOEnabled,
- directMemoryEnabled,
- memoryForCache,
- numLocalDirs);
- } catch (Exception e) {
- LOG.error("Unable to create MiniLlapCluster. Exception: " + e.getMessage());
- throw new IOException(e);
- }
}
@Override
@@ -458,15 +407,6 @@ public int getJobTrackerPort() throws UnsupportedOperationException {
@Override
public void shutdown() throws IOException {
mr.stop();
-
- if (miniLlapKlass != null && miniLlapCluster != null) {
- try {
- Method stop = miniLlapKlass.getMethod("stop", new Class[]{});
- stop.invoke(miniLlapCluster);
- } catch (Exception e) {
- LOG.error("Unable to stop llap daemon. Exception: " + e.getMessage());
- }
- }
}
@Override
diff --git shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
index 7a5a9b5..a44d0c0 100644
--- shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
+++ shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
@@ -90,7 +90,7 @@ public MiniMrShim getMiniMrCluster(Configuration conf, int numberOfTaskTrackers,
String nameNode, int numDir) throws IOException;
public MiniMrShim getMiniTezCluster(Configuration conf, int numberOfTaskTrackers,
- String nameNode, boolean isLlap) throws IOException;
+ String nameNode) throws IOException;
public MiniMrShim getMiniSparkCluster(Configuration conf, int numberOfTaskTrackers,
String nameNode, int numDir) throws IOException;