diff --git data/conf/llap/hive-site.xml data/conf/llap/hive-site.xml index c2bef58..72bdcfb 100644 --- data/conf/llap/hive-site.xml +++ data/conf/llap/hive-site.xml @@ -273,4 +273,48 @@ false + + + + hive.llap.daemon.service.hosts + localhost + + + + hive.llap.daemon.service.port + 0 + + + + hive.llap.daemon.num.executors + 4 + + + + hive.llap.daemon.task.scheduler.wait.queue.size + 4 + + + + hive.llap.cache.allow.synthetic.fileid + true + + + + + ipc.client.low-latency + true + + + + ipc.client.tcpnodelay + true + + + + ipc.clients-per-factory + 4 + + + diff --git data/conf/llap/llap-daemon-site.xml data/conf/llap/llap-daemon-site.xml deleted file mode 100644 index 98c0f2b..0000000 --- data/conf/llap/llap-daemon-site.xml +++ /dev/null @@ -1,61 +0,0 @@ - - - - - - hive.llap.daemon.service.hosts - localhost - - - - hive.llap.daemon.service.port - 0 - - - - hive.llap.daemon.num.executors - 4 - - - - hive.llap.daemon.task.scheduler.wait.queue.size - 4 - - - - hive.llap.cache.allow.synthetic.fileid - true - - - - - ipc.client.low-latency - true - - - - ipc.client.tcpnodelay - true - - - - ipc.clients-per-factory - 4 - - - diff --git itests/hive-unit/pom.xml itests/hive-unit/pom.xml index 97786d9..7219f1d 100644 --- itests/hive-unit/pom.xml +++ itests/hive-unit/pom.xml @@ -75,6 +75,11 @@ hive-hcatalog-streaming ${project.version} + + org.apache.hive + hive-it-util + ${project.version} + org.apache.hadoop @@ -123,12 +128,6 @@ org.apache.hive - hive-it-util - ${project.version} - test - - - org.apache.hive hive-jdbc ${project.version} test diff --git itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java index 751d8ea..26caa8d 100644 --- itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java +++ itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java @@ -31,6 +31,8 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.llap.LlapItUtils; +import org.apache.hadoop.hive.llap.daemon.MiniLlapCluster; import org.apache.hadoop.hive.metastore.MetaStoreUtils; import org.apache.hadoop.hive.ql.WindowsPathUtil; import org.apache.hadoop.hive.ql.exec.Utilities; @@ -59,6 +61,7 @@ private static final AtomicLong hs2Counter = new AtomicLong(); private MiniMrShim mr; private MiniDFSShim dfs; + private MiniLlapCluster llapCluster = null; private final FileSystem localFS; private boolean useMiniKdc = false; private final String serverPrincipal; @@ -186,13 +189,15 @@ private MiniHS2(HiveConf hiveConf, MiniClusterType miniClusterType, boolean useM // Initialize the execution engine based on cluster type switch (miniClusterType) { case TEZ: - mr = ShimLoader.getHadoopShims().getMiniTezCluster(hiveConf, 4, uriString, false); + mr = ShimLoader.getHadoopShims().getMiniTezCluster(hiveConf, 4, uriString); break; case LLAP: if (usePortsFromConf) { hiveConf.setBoolean("minillap.usePortsFromConf", true); } - mr = ShimLoader.getHadoopShims().getMiniTezCluster(hiveConf, 4, uriString, true); + llapCluster = LlapItUtils.startAndGetMiniLlapCluster(hiveConf, null); + + mr = ShimLoader.getHadoopShims().getMiniTezCluster(hiveConf, 4, uriString); break; case MR: mr = ShimLoader.getHadoopShims().getMiniMrCluster(hiveConf, 4, uriString, 1); @@ -284,6 +289,9 @@ public void stop() { hiveServer2.stop(); setStarted(false); try { + if (llapCluster != null) { + llapCluster.stop(); + } if (mr != null) { mr.shutdown(); mr = null; diff --git itests/util/pom.xml itests/util/pom.xml index aaafc0a..4789586 100644 --- itests/util/pom.xml +++ itests/util/pom.xml @@ -97,6 +97,17 @@ tests + org.apache.hive + hive-llap-server + ${project.version} + + + org.apache.hive + hive-llap-server + ${project.version} + test-jar + + org.apache.hadoop hadoop-common ${hadoop.version} diff --git itests/util/src/main/java/org/apache/hadoop/hive/llap/LlapItUtils.java itests/util/src/main/java/org/apache/hadoop/hive/llap/LlapItUtils.java new file mode 100644 index 0000000..cb4aba5 --- /dev/null +++ itests/util/src/main/java/org/apache/hadoop/hive/llap/LlapItUtils.java @@ -0,0 +1,84 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.llap; + +import java.io.File; +import java.io.IOException; +import java.net.URL; +import java.util.Iterator; +import java.util.Map; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.llap.configuration.LlapDaemonConfiguration; +import org.apache.hadoop.hive.llap.daemon.MiniLlapCluster; +import org.apache.hadoop.hive.llap.daemon.impl.LlapDaemon; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class LlapItUtils { + + private static final Logger LOG = LoggerFactory.getLogger(LlapItUtils.class); + + public static MiniLlapCluster startAndGetMiniLlapCluster(Configuration conf, String confDir) throws + IOException { + MiniLlapCluster llapCluster; + LOG.info("Using conf dir: {}", confDir); + if (confDir != null && !confDir.isEmpty()) { + conf.addResource(new URL("file://" + new File(confDir).toURI().getPath() + + "/tez-site.xml")); + } + + Configuration daemonConf = new LlapDaemonConfiguration(conf); + final String clusterName = "llap"; + final long maxMemory = LlapDaemon.getTotalHeapSize(); + // 15% for io cache + final long memoryForCache = (long) (0.15f * maxMemory); + // 75% for 4 executors + final long totalExecutorMemory = (long) (0.75f * maxMemory); + final int numExecutors = HiveConf.getIntVar(conf, HiveConf.ConfVars.LLAP_DAEMON_NUM_EXECUTORS); + final boolean asyncIOEnabled = true; + // enabling this will cause test failures in Mac OS X + final boolean directMemoryEnabled = false; + final int numLocalDirs = 1; + LOG.info("MiniLlap Configs - maxMemory: " + maxMemory + " memoryForCache: " + memoryForCache + + " totalExecutorMemory: " + totalExecutorMemory + " numExecutors: " + numExecutors + + " asyncIOEnabled: " + asyncIOEnabled + " directMemoryEnabled: " + directMemoryEnabled + + " numLocalDirs: " + numLocalDirs); + llapCluster = MiniLlapCluster.create(clusterName, + numExecutors, + totalExecutorMemory, + asyncIOEnabled, + directMemoryEnabled, + memoryForCache, + numLocalDirs); + llapCluster.init(daemonConf); + llapCluster.start(); + + // Augment conf with the settings from the started llap configuration. + Configuration llapConf = llapCluster.getClusterSpecificConfiguration(); + Iterator> confIter = llapConf.iterator(); + while (confIter.hasNext()) { + Map.Entry entry = confIter.next(); + conf.set(entry.getKey(), entry.getValue()); + } + return llapCluster; + } + +} diff --git itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index 5e81e98..5ccbcba 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -51,6 +51,7 @@ import java.util.Comparator; import java.util.Deque; import java.util.HashSet; +import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -82,6 +83,10 @@ import org.apache.hadoop.hive.common.io.SortPrintStream; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.llap.LlapItUtils; +import org.apache.hadoop.hive.llap.configuration.LlapDaemonConfiguration; +import org.apache.hadoop.hive.llap.daemon.MiniLlapCluster; +import org.apache.hadoop.hive.llap.daemon.impl.LlapDaemon; import org.apache.hadoop.hive.llap.io.api.LlapProxy; import org.apache.hadoop.hive.metastore.MetaStoreUtils; import org.apache.hadoop.hive.metastore.api.Index; @@ -166,6 +171,7 @@ private HadoopShims.MiniMrShim mr = null; private HadoopShims.MiniDFSShim dfs = null; private HadoopShims.HdfsEncryptionShim hes = null; + private MiniLlapCluster llapCluster = null; private String hadoopVer = null; private QTestSetup setup = null; private TezSessionState tezSessionState = null; @@ -429,15 +435,10 @@ public QTestUtil(String outDir, String logDir, MiniClusterType clusterType, conf.addResource(new URL("file://" + new File(confDir).toURI().getPath() + "/tez-site.xml")); } - mr = shims.getMiniTezCluster(conf, 4, uriString, false); + mr = shims.getMiniTezCluster(conf, 4, uriString); } else if (clusterType == MiniClusterType.llap) { - if (confDir != null && !confDir.isEmpty()) { - conf.addResource(new URL("file://" + new File(confDir).toURI().getPath() - + "/tez-site.xml")); - conf.addResource(new URL("file://" + new File(confDir).toURI().getPath() - + "/llap-daemon-site.xml")); - } - mr = shims.getMiniTezCluster(conf, 2, uriString, true); + llapCluster = LlapItUtils.startAndGetMiniLlapCluster(conf, confDir); + mr = shims.getMiniTezCluster(conf, 2, uriString); } else if (clusterType == MiniClusterType.miniSparkOnYarn) { mr = shims.getMiniSparkCluster(conf, 4, uriString, 1); } else { diff --git llap-server/src/java/org/apache/hadoop/hive/llap/configuration/LlapDaemonConfiguration.java llap-server/src/java/org/apache/hadoop/hive/llap/configuration/LlapDaemonConfiguration.java index 51e8509..88f3b19 100644 --- llap-server/src/java/org/apache/hadoop/hive/llap/configuration/LlapDaemonConfiguration.java +++ llap-server/src/java/org/apache/hadoop/hive/llap/configuration/LlapDaemonConfiguration.java @@ -14,6 +14,7 @@ package org.apache.hadoop.hive.llap.configuration; +import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.common.classification.InterfaceAudience; import org.apache.hadoop.hive.conf.HiveConf; @@ -21,6 +22,7 @@ /** * Configuration for LLAP daemon processes only. This should not be used by any clients. */ +@InterfaceAudience.Private public class LlapDaemonConfiguration extends Configuration { @InterfaceAudience.Private @@ -46,4 +48,10 @@ public LlapDaemonConfiguration() { } addResource(LLAP_DAEMON_SITE); } + + @VisibleForTesting + public LlapDaemonConfiguration(Configuration conf) { + this(); + addResource(conf); + } } \ No newline at end of file diff --git shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java index 9a3a31c..e028212 100644 --- shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java +++ shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java @@ -370,8 +370,8 @@ public void setupConfiguration(Configuration conf) { */ @Override public MiniMrShim getMiniTezCluster(Configuration conf, int numberOfTaskTrackers, - String nameNode, boolean isLlap) throws IOException { - return new MiniTezShim(conf, numberOfTaskTrackers, nameNode, isLlap); + String nameNode) throws IOException { + return new MiniTezShim(conf, numberOfTaskTrackers, nameNode); } /** @@ -381,11 +381,8 @@ public MiniMrShim getMiniTezCluster(Configuration conf, int numberOfTaskTrackers private final MiniTezCluster mr; private final Configuration conf; - private Class miniLlapKlass; - private Object miniLlapCluster; - public MiniTezShim(Configuration conf, int numberOfTaskTrackers, String nameNode, - boolean isLlap) throws IOException { + public MiniTezShim(Configuration conf, int numberOfTaskTrackers, String nameNode) throws IOException { mr = new MiniTezCluster("hive", numberOfTaskTrackers); conf.set("fs.defaultFS", nameNode); conf.set("tez.am.log.level", "DEBUG"); @@ -393,54 +390,6 @@ public MiniTezShim(Configuration conf, int numberOfTaskTrackers, String nameNode mr.init(conf); mr.start(); this.conf = mr.getConfig(); - if (isLlap) { - createAndLaunchLlapDaemon(this.conf); - } else { - miniLlapCluster = null; - } - } - - private void createAndLaunchLlapDaemon(final Configuration conf) - throws IOException { - try { - final String clusterName = "llap"; - Class llapDaemonKlass = - Class.forName("org.apache.hadoop.hive.llap.daemon.impl.LlapDaemon", - false, ShimLoader.class.getClassLoader()); - Method totalMemMethod = llapDaemonKlass.getMethod("getTotalHeapSize"); - final long maxMemory = (long) totalMemMethod.invoke(null); - // 15% for io cache - final long memoryForCache = (long) (0.15f * maxMemory); - // 75% for executors - final long totalExecutorMemory = (long) (0.75f * maxMemory); - final int numExecutors = conf.getInt("llap.daemon.num.executors", 4); - final boolean asyncIOEnabled = true; - // enabling this will cause test failures in Mac OS X - final boolean directMemoryEnabled = false; - final int numLocalDirs = 1; - LOG.info("MiniLlap Configs - maxMemory: " + maxMemory + " memoryForCache: " + memoryForCache - + " totalExecutorMemory: " + totalExecutorMemory + " numExecutors: " + numExecutors - + " asyncIOEnabled: " + asyncIOEnabled + " directMemoryEnabled: " + directMemoryEnabled - + " numLocalDirs: " + numLocalDirs); - - miniLlapKlass = Class.forName("org.apache.hadoop.hive.llap.daemon.MiniLlapCluster", - false, ShimLoader.class.getClassLoader()); - Method create = miniLlapKlass.getMethod("createAndLaunch", new Class[]{Configuration.class, - String.class, Integer.TYPE, Long.TYPE, Boolean.TYPE, Boolean.TYPE, - Long.TYPE, Integer.TYPE}); - miniLlapCluster = create.invoke(null, - conf, - clusterName, - numExecutors, - totalExecutorMemory, - asyncIOEnabled, - directMemoryEnabled, - memoryForCache, - numLocalDirs); - } catch (Exception e) { - LOG.error("Unable to create MiniLlapCluster. Exception: " + e.getMessage()); - throw new IOException(e); - } } @Override @@ -458,15 +407,6 @@ public int getJobTrackerPort() throws UnsupportedOperationException { @Override public void shutdown() throws IOException { mr.stop(); - - if (miniLlapKlass != null && miniLlapCluster != null) { - try { - Method stop = miniLlapKlass.getMethod("stop", new Class[]{}); - stop.invoke(miniLlapCluster); - } catch (Exception e) { - LOG.error("Unable to stop llap daemon. Exception: " + e.getMessage()); - } - } } @Override diff --git shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java index 7a5a9b5..a44d0c0 100644 --- shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java +++ shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java @@ -90,7 +90,7 @@ public MiniMrShim getMiniMrCluster(Configuration conf, int numberOfTaskTrackers, String nameNode, int numDir) throws IOException; public MiniMrShim getMiniTezCluster(Configuration conf, int numberOfTaskTrackers, - String nameNode, boolean isLlap) throws IOException; + String nameNode) throws IOException; public MiniMrShim getMiniSparkCluster(Configuration conf, int numberOfTaskTrackers, String nameNode, int numDir) throws IOException;