diff --git itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniLlapLocalCliDriver.java itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniLlapLocalCliDriver.java new file mode 100644 index 0000000..5b1caf6 --- /dev/null +++ itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniLlapLocalCliDriver.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.cli; + +import java.io.File; +import java.util.List; + +import org.apache.hadoop.hive.cli.control.CliAdapter; +import org.apache.hadoop.hive.cli.control.CliConfigs; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestRule; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +@RunWith(Parameterized.class) +public class TestMiniLlapLocalCliDriver { + + static CliAdapter adapter = new CliConfigs.MiniLlapLocalCliConfig().getCliAdapter(); + + @Parameters(name = "{0}") + public static List getParameters() throws Exception { + return adapter.getParameters(); + } + + @ClassRule + public static TestRule cliClassRule = adapter.buildClassRule(); + + @Rule + public TestRule cliTestRule = adapter.buildTestRule(); + + private String name; + private File qfile; + + public TestMiniLlapLocalCliDriver(String name, File qfile) { + this.name = name; + this.qfile = qfile; + } + + @Test + public void testCliDriver() throws Exception { + adapter.runTest(name, qfile); + } + +} diff --git itests/src/test/resources/testconfiguration.properties itests/src/test/resources/testconfiguration.properties index e810a58..364898f 100644 --- itests/src/test/resources/testconfiguration.properties +++ itests/src/test/resources/testconfiguration.properties @@ -54,8 +54,40 @@ minitez.query.files=explainuser_3.q,\ stats_filemetadata.q,\ tez_union_with_udf.q + minillap.shared.query.files=acid_globallimit.q,\ - alter_merge_2_orc.q,\ + insert_into1.q,\ + insert_into2.q,\ + insert_values_orig_table.,\ + llapdecider.q,\ + mapreduce1.q,\ + mapreduce2.q,\ + orc_merge1.q,\ + orc_merge10.q,\ + orc_merge2.q,\ + orc_merge3.q,\ + orc_merge4.q,\ + orc_merge_diff_fs.q,\ + script_env_var1.q,\ + script_env_var2.q,\ + script_pipe.q,\ + tez_union_view.q,\ + transform1.q,\ + transform2.q,\ + transform_ppr1.q,\ + transform_ppr2.q,\ + unionDistinct_1.q,\ + union_type_chk.q,\ + orc_ppd_basic.q,\ + orc_ppd_schema_evol_3a.q,\ + cte_2.q,\ + cte_4.q,\ + llap_nullscan.q,\ + dynamic_partition_pruning_2.q,\ + tez_union_dynamic_partition.q,\ + union_fast_stats.q + +minillaplocal.shared.query.files=alter_merge_2_orc.q,\ alter_merge_orc.q,\ alter_merge_stats_orc.q,\ auto_join0.q,\ @@ -111,9 +143,7 @@ minillap.shared.query.files=acid_globallimit.q,\ cross_product_check_2.q,\ ctas.q,\ cte_1.q,\ - cte_2.q,\ cte_3.q,\ - cte_4.q,\ cte_5.q,\ cte_mat_1.q,\ cte_mat_2.q,\ @@ -131,7 +161,6 @@ minillap.shared.query.files=acid_globallimit.q,\ delete_whole_partition.q,\ disable_merge_for_bucketing.q,\ dynamic_partition_pruning.q,\ - dynamic_partition_pruning_2.q,\ dynpart_sort_opt_vectorization.q,\ dynpart_sort_optimization.q,\ dynpart_sort_optimization2.q,\ @@ -145,13 +174,10 @@ minillap.shared.query.files=acid_globallimit.q,\ having.q,\ identity_project_remove_skip.q,\ insert1.q,\ - insert_into1.q,\ - insert_into2.q,\ insert_orig_table.q,\ insert_update_delete.q,\ insert_values_dynamic_partitioned.q,\ insert_values_non_partitioned.q,\ - insert_values_orig_table.,\ insert_values_partitioned.q,\ insert_values_tmp_table.q,\ join0.q,\ @@ -159,8 +185,6 @@ minillap.shared.query.files=acid_globallimit.q,\ join_nullsafe.q,\ leftsemijoin.q,\ limit_pushdown.q,\ - llap_nullscan.q,\ - llapdecider.q,\ load_dyn_part1.q,\ load_dyn_part2.q,\ load_dyn_part3.q,\ @@ -168,8 +192,6 @@ minillap.shared.query.files=acid_globallimit.q,\ mapjoin2.q,\ mapjoin_decimal.q,\ mapjoin_mapjoin.q,\ - mapreduce1.q,\ - mapreduce2.q,\ merge1.q,\ merge2.q,\ mergejoin.q,\ @@ -180,36 +202,25 @@ minillap.shared.query.files=acid_globallimit.q,\ nonmr_fetch_threshold.q,\ optimize_nullscan.q,\ orc_analyze.q,\ - orc_merge1.q,\ - orc_merge10.q,\ orc_merge11.q,\ - orc_merge2.q,\ - orc_merge3.q,\ - orc_merge4.q,\ orc_merge5.q,\ orc_merge6.q,\ orc_merge7.q,\ orc_merge8.q,\ orc_merge9.q,\ - orc_merge_diff_fs.q,\ orc_merge_incompat1.q,\ orc_merge_incompat2.q,\ orc_merge_incompat3.q,\ - orc_ppd_basic.q,\ orc_ppd_schema_evol_1a.q,\ orc_ppd_schema_evol_1b.q,\ orc_ppd_schema_evol_2a.q,\ orc_ppd_schema_evol_2b.q,\ - orc_ppd_schema_evol_3a.q,\ order_null.q,\ parallel.q,\ ptf.q,\ ptf_matchpath.q,\ ptf_streaming.q,\ sample1.q,\ - script_env_var1.q,\ - script_env_var2.q,\ - script_pipe.q,\ selectDistinctStar.q,\ select_dummy_source.q,\ skewjoin.q,\ @@ -237,16 +248,10 @@ minillap.shared.query.files=acid_globallimit.q,\ tez_union.q,\ tez_union2.q,\ tez_union_decimal.q,\ - tez_union_dynamic_partition.q,\ tez_union_group_by.q,\ tez_union_multiinsert.q,\ - tez_union_view.q,\ tez_vector_dynpart_hashjoin_1.q,\ tez_vector_dynpart_hashjoin_2.q,\ - transform1.q,\ - transform2.q,\ - transform_ppr1.q,\ - transform_ppr2.q,\ union2.q,\ union3.q,\ union4.q,\ @@ -255,10 +260,7 @@ minillap.shared.query.files=acid_globallimit.q,\ union7.q,\ union8.q,\ union9.q,\ - unionDistinct_1.q,\ - union_fast_stats.q,\ union_stats.q,\ - union_type_chk.q,\ update_after_multiple_inserts.q,\ update_all_non_partitioned.q,\ update_all_partitioned.q,\ @@ -402,12 +404,36 @@ minillap.shared.query.files=acid_globallimit.q,\ vectorized_timestamp_funcs.q,\ vectorized_timestamp_ints_casts.q -minillap.query.files=acid_bucket_pruning.q,\ - acid_vectorization_missing_cols.q,\ - auto_sortmerge_join_16.q,\ - bucket4.q,\ +minillap.query.files=explainuser_2.q,\ bucket5.q,\ bucket6.q,\ + empty_dir_in_table.q,\ + llap_udf.q,\ + llapdecider.q,\ + reduce_deduplicate.q,\ + remote_script.q,\ + tez_aggr_part_stats.q,\ + tez_union_view.q,\ + file_with_header_footer.q,\ + external_table_with_space_in_location_path.q,\ + import_exported_table.q,\ + orc_llap_counters.q,\ + orc_llap_counters1.q,\ + load_hdfs_file_with_space_in_the_name.q,\ + orc_ppd_basic.q,\ + schemeAuthority.q,\ + schemeAuthority2.q,\ + temp_table_external.q,\ + table_nonprintable.q,\ + llap_nullscan.q,\ + acid_bucket_pruning.q,\ + dynamic_partition_pruning_2.q,\ + tez_union_dynamic_partition.q,\ + load_fs2.q + +minillaplocal.query.files=acid_vectorization_missing_cols.q,\ + auto_sortmerge_join_16.q,\ + bucket4.q,\ bucket_many.q,\ bucket_map_join_tez1.q,\ bucket_map_join_tez2.q,\ @@ -418,18 +444,12 @@ minillap.query.files=acid_bucket_pruning.q,\ constprog_dpp.q,\ disable_merge_for_bucketing.q,\ dynamic_partition_pruning.q,\ - dynamic_partition_pruning_2.q,\ - empty_dir_in_table.q,\ exchgpartition2lel.q,\ explainuser_1.q,\ - explainuser_2.q,\ explainuser_4.q,\ - external_table_with_space_in_location_path.q,\ - file_with_header_footer.q,\ groupby2.q,\ hybridgrace_hashjoin_1.q,\ hybridgrace_hashjoin_2.q,\ - import_exported_table.q,\ infer_bucket_sort_bucketed_table.q,\ input16_cc.q,\ insert_dir_distcp.q,\ @@ -437,24 +457,14 @@ minillap.query.files=acid_bucket_pruning.q,\ join_acid_non_acid.q,\ leftsemijoin_mr.q,\ list_bucket_dml_10.q,\ - llap_nullscan.q,\ - llap_udf.q,\ - llapdecider.q,\ - load_fs2.q,\ - load_hdfs_file_with_space_in_the_name.q,\ lvj_mapjoin.q,\ mapjoin_decimal.q,\ mergejoin_3way.q,\ mrr.q,\ non_native_window_udf.q,\ orc_llap.q,\ - orc_llap_counters.q,\ - orc_llap_counters1.q,\ orc_llap_nonvector.q,\ - orc_ppd_basic.q,\ quotedid_smb.q,\ - reduce_deduplicate.q,\ - remote_script.q,\ schema_evol_orc_acid_part.q,\ schema_evol_orc_acid_part_update.q,\ schema_evol_orc_acid_table.q,\ @@ -484,12 +494,7 @@ minillap.query.files=acid_bucket_pruning.q,\ schema_evol_text_vecrow_part_all_complex.q,\ schema_evol_text_vecrow_part_all_primitive.q,\ schema_evol_text_vecrow_table.q,\ - schemeAuthority.q,\ - schemeAuthority2.q,\ smb_cache.q,\ - table_nonprintable.q,\ - temp_table_external.q,\ - tez_aggr_part_stats.q,\ tez_bmj_schema_evolution.q,\ tez_dml.q,\ tez_dynpart_hashjoin_1.q,\ @@ -510,10 +515,8 @@ minillap.query.files=acid_bucket_pruning.q,\ tez_union.q,\ tez_union2.q,\ tez_union_decimal.q,\ - tez_union_dynamic_partition.q,\ tez_union_group_by.q,\ tez_union_multiinsert.q,\ - tez_union_view.q,\ tez_vector_dynpart_hashjoin_1.q,\ tez_vector_dynpart_hashjoin_2.q,\ uber_reduce.q,\ diff --git itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java index 03d4075..c12f51e 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java +++ itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java @@ -409,7 +409,18 @@ public CliAdapter getCliAdapter() { } protected void setMetastoreType(MetastoreType mt) { - metastoreType=mt; + String metaStoreTypeProperty = getSysPropValue("metaStoreType"); + if (metaStoreTypeProperty != null) { + if (metaStoreTypeProperty.equalsIgnoreCase("sql")) { + metastoreType = MetastoreType.sql; + } else if (metaStoreTypeProperty.equalsIgnoreCase("hbase")) { + metastoreType = MetastoreType.hbase; + } else { + throw new IllegalArgumentException("Unknown metastore type: " + metaStoreTypeProperty); + } + } else { + metastoreType = mt; + } } public MetastoreType getMetastoreType() { diff --git itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java index ca72282..201c3d3 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java +++ itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java @@ -47,6 +47,7 @@ public CliConfig() { setQueryDir("ql/src/test/queries/clientpositive"); excludesFrom(testConfigProps, "minillap.query.files"); + excludesFrom(testConfigProps, "minillaplocal.query.files"); excludesFrom(testConfigProps, "minimr.query.files"); excludesFrom(testConfigProps, "minitez.query.files"); excludesFrom(testConfigProps, "encrypted.query.files"); @@ -129,6 +130,7 @@ public MiniTezCliConfig() { setHiveConfDir("data/conf/tez"); setClusterType(MiniClusterType.tez); setMetastoreType(MetastoreType.hbase); + setFsType(QTestUtil.FsType.hdfs); } catch (Exception e) { throw new RuntimeException("can't construct cliconfig", e); } @@ -152,6 +154,33 @@ public MiniLlapCliConfig() { setHiveConfDir("data/conf/llap"); setClusterType(MiniClusterType.llap); + setMetastoreType(MetastoreType.sql); + } catch (Exception e) { + throw new RuntimeException("can't construct cliconfig", e); + } + } + } + + public static class MiniLlapLocalCliConfig extends AbstractCliConfig { + + public MiniLlapLocalCliConfig() { + super(CoreCliDriver.class); + try { + setQueryDir("ql/src/test/queries/clientpositive"); + + includesFrom(testConfigProps, "minillaplocal.query.files"); + includesFrom(testConfigProps, "minillaplocal.shared.query.files"); + + setResultsDir("ql/src/test/results/clientpositive/llap"); + setLogDir("itests/qtest/target/qfile-results/clientpositive"); + + setInitScript("q_test_init.sql"); + setCleanupScript("q_test_cleanup.sql"); + + setHiveConfDir("data/conf/llap"); + setClusterType(MiniClusterType.llap_local); + setMetastoreType(MetastoreType.sql); + setFsType(QTestUtil.FsType.local); } catch (Exception e) { throw new RuntimeException("can't construct cliconfig", e); } diff --git itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java index d83ff45..a735346 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java +++ itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java @@ -61,7 +61,7 @@ public void beforeClass() { @Override public QTestUtil invokeInternal() throws Exception { return new QTestUtil((cliConfig.getResultsDir()), (cliConfig.getLogDir()), miniMR, - hiveConfDir, hadoopVer, initScript, cleanupScript, useHBaseMetastore, true, false, + hiveConfDir, hadoopVer, initScript, cleanupScript, useHBaseMetastore, true, cliConfig.getFsType()); } }.invoke("QtestUtil instance created", LOG, true); diff --git itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index 909d7f6..ebc93a9 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -50,6 +50,7 @@ import java.util.Collection; import java.util.Comparator; import java.util.Deque; +import java.util.EnumSet; import java.util.HashSet; import java.util.LinkedList; import java.util.List; @@ -152,7 +153,6 @@ private String testWarehouse; private final String testFiles; - private final boolean localMode; protected final String outDir; protected final String logDir; private final TreeMap qMap; @@ -411,6 +411,11 @@ private void createRemoteDirs() { } } + private enum CoreClusteType { + mr, + tez, + spark + } public enum FsType { local, @@ -420,35 +425,48 @@ private void createRemoteDirs() { public enum MiniClusterType { - mr(FsType.hdfs), - tez(FsType.hdfs), - spark(FsType.local), - miniSparkOnYarn(FsType.hdfs), - llap(FsType.hdfs), - none(FsType.local); + mr(CoreClusteType.mr, FsType.hdfs), + tez(CoreClusteType.tez, FsType.hdfs), + tez_local(CoreClusteType.tez, FsType.local), + spark(CoreClusteType.spark, FsType.local), + miniSparkOnYarn(CoreClusteType.spark, FsType.hdfs), + llap(CoreClusteType.tez, FsType.hdfs), + llap_local(CoreClusteType.tez, FsType.local), + none(CoreClusteType.mr, FsType.local); + private final CoreClusteType coreClusterType; private final FsType defaultFsType; - MiniClusterType(FsType defaultFsType) { + MiniClusterType(CoreClusteType coreClusterType, FsType defaultFsType) { + this.coreClusterType = coreClusterType; this.defaultFsType = defaultFsType; } + public CoreClusteType getCoreClusterType() { + return coreClusterType; + } + public FsType getDefaultFsType() { return defaultFsType; } public static MiniClusterType valueForString(String type) { + // Replace this with valueOf. if (type.equals("miniMR")) { return mr; } else if (type.equals("tez")) { return tez; + } else if (type.equals("tez_local")) { + return tez_local; } else if (type.equals("spark")) { return spark; } else if (type.equals("miniSparkOnYarn")) { return miniSparkOnYarn; } else if (type.equals("llap")) { return llap; + } else if (type.equals("llap_local")) { + return llap_local; } else { return none; } @@ -487,25 +505,24 @@ public QTestUtil(String outDir, String logDir, MiniClusterType clusterType, String confDir, String hadoopVer, String initScript, String cleanupScript, boolean useHBaseMetastore, boolean withLlapIo) throws Exception { this(outDir, logDir, clusterType, confDir, hadoopVer, initScript, cleanupScript, - useHBaseMetastore, withLlapIo, false, null); + useHBaseMetastore, withLlapIo, null); } public QTestUtil(String outDir, String logDir, MiniClusterType clusterType, String confDir, String hadoopVer, String initScript, String cleanupScript, - boolean useHBaseMetastore, boolean withLlapIo, boolean localMode, FsType fsType) + boolean useHBaseMetastore, boolean withLlapIo, FsType fsType) throws Exception { LOG.info("Setting up QTestUtil with outDir={}, logDir={}, clusterType={}, confDir={}," + " hadoopVer={}, initScript={}, cleanupScript={}, useHbaseMetaStore={}, withLlapIo={}," + - " localMode={}, fsType={}" + " fsType={}" , outDir, logDir, clusterType, confDir, hadoopVer, initScript, cleanupScript, - useHBaseMetastore, withLlapIo, localMode, fsType); + useHBaseMetastore, withLlapIo, fsType); Preconditions.checkNotNull(clusterType, "ClusterType cannot be null"); if (fsType != null) { this.fsType = fsType; } else { this.fsType = clusterType.getDefaultFsType(); } - this.localMode = localMode; this.outDir = outDir; this.logDir = logDir; this.useHBaseMetastore = useHBaseMetastore; @@ -604,28 +621,22 @@ private void setupFileSystem(HadoopShims shims) throws IOException { private void setupMiniCluster(HadoopShims shims, String confDir) throws IOException { - if (localMode) { - Preconditions - .checkState(clusterType == MiniClusterType.tez || clusterType == MiniClusterType.llap, - "localMode can currently only be set for tez or llap"); - } - String uriString = WindowsPathUtil.getHdfsUriString(fs.getUri().toString()); - if (clusterType == MiniClusterType.tez || clusterType == MiniClusterType.llap) { + if (clusterType.getCoreClusterType() == CoreClusteType.tez) { if (confDir != null && !confDir.isEmpty()) { conf.addResource(new URL("file://" + new File(confDir).toURI().getPath() + "/tez-site.xml")); } int numTrackers; - if (clusterType == MiniClusterType.tez) { - numTrackers = 4; - } else { + if (EnumSet.of(MiniClusterType.llap, MiniClusterType.llap_local).contains(clusterType)) { llapCluster = LlapItUtils.startAndGetMiniLlapCluster(conf, setup.zooKeeperCluster, confDir); numTrackers = 2; + } else { + numTrackers = 4; } - if (localMode) { - mr = shims.getLocalMiniTezCluster(conf, clusterType == MiniClusterType.llap); + if (EnumSet.of(MiniClusterType.llap_local, MiniClusterType.tez_local).contains(clusterType)) { + mr = shims.getLocalMiniTezCluster(conf, clusterType == MiniClusterType.llap_local); } else { mr = shims.getMiniTezCluster(conf, numTrackers, uriString); } @@ -642,7 +653,7 @@ public void shutdown() throws Exception { cleanUp(); } - if (clusterType == MiniClusterType.tez || clusterType == MiniClusterType.llap) { + if (clusterType.getCoreClusterType() == CoreClusteType.tez) { SessionState.get().getTezSession().close(false); } setup.tearDown(); @@ -1152,8 +1163,7 @@ public String cliInit(String tname, boolean recreate) throws Exception { SessionState oldSs = SessionState.get(); boolean canReuseSession = !qNoSessionReuseQuerySet.contains(tname); - if (oldSs != null && canReuseSession - && (clusterType == MiniClusterType.tez || clusterType == MiniClusterType.llap)) { + if (oldSs != null && canReuseSession && clusterType.getCoreClusterType() == CoreClusteType.tez) { // Copy the tezSessionState from the old CliSessionState. tezSessionState = oldSs.getTezSession(); oldSs.setTezSession(null); @@ -1161,8 +1171,7 @@ public String cliInit(String tname, boolean recreate) throws Exception { oldSs.close(); } - if (oldSs != null && (clusterType == MiniClusterType.spark - || clusterType == MiniClusterType.miniSparkOnYarn)) { + if (oldSs != null && clusterType.getCoreClusterType() == CoreClusteType.spark) { sparkSession = oldSs.getSparkSession(); ss.setSparkSession(sparkSession); oldSs.setSparkSession(null); @@ -1225,8 +1234,7 @@ private CliSessionState startSessionState(boolean canReuseSession) ss.err = System.out; SessionState oldSs = SessionState.get(); - if (oldSs != null && canReuseSession - && (clusterType == MiniClusterType.tez || clusterType == MiniClusterType.llap)) { + if (oldSs != null && canReuseSession && clusterType.getCoreClusterType() == CoreClusteType.tez) { // Copy the tezSessionState from the old CliSessionState. tezSessionState = oldSs.getTezSession(); ss.setTezSession(tezSessionState); @@ -1234,8 +1242,7 @@ private CliSessionState startSessionState(boolean canReuseSession) oldSs.close(); } - if (oldSs != null && (clusterType == MiniClusterType.spark - || clusterType == MiniClusterType.miniSparkOnYarn)) { + if (oldSs != null && clusterType.getCoreClusterType() == CoreClusteType.spark) { sparkSession = oldSs.getSparkSession(); ss.setSparkSession(sparkSession); oldSs.setSparkSession(null); diff --git llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java index 53a2295..752e6ee 100644 --- llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java +++ llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java @@ -116,8 +116,6 @@ public LlapDaemon(Configuration daemonConf, int numExecutors, long executorMemor int mngPort, int shufflePort, int webPort, String appName) { super("LlapDaemon"); - initializeLogging(daemonConf); - printAsciiArt(); Preconditions.checkArgument(numExecutors > 0); @@ -278,7 +276,7 @@ public LlapDaemon(Configuration daemonConf, int numExecutors, long executorMemor addIfService(amReporter); } - private void initializeLogging(final Configuration conf) { + private static void initializeLogging(final Configuration conf) { long start = System.currentTimeMillis(); URL llap_l4j2 = LlapDaemon.class.getClassLoader().getResource( LlapConstants.LOG4j2_PROPERTIES_FILE); @@ -450,6 +448,7 @@ public static void main(String[] args) throws Exception { long ioMemoryBytes = HiveConf.getSizeVar(daemonConf, ConfVars.LLAP_IO_MEMORY_MAX_SIZE); boolean isDirectCache = HiveConf.getBoolVar(daemonConf, ConfVars.LLAP_ALLOCATOR_DIRECT); boolean isLlapIo = HiveConf.getBoolVar(daemonConf, HiveConf.ConfVars.LLAP_IO_ENABLED, true); + LlapDaemon.initializeLogging(daemonConf); llapDaemon = new LlapDaemon(daemonConf, numExecutors, executorMemoryBytes, isLlapIo, isDirectCache, ioMemoryBytes, localDirs, rpcPort, mngPort, shufflePort, webPort, appName); diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java index 1ab7137..f46e2d7 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java @@ -156,6 +156,7 @@ private void convertWork(TezWork tezWork, BaseWork work) if (tezWork.getChildren(work).isEmpty() && work instanceof ReduceWork && ((ReduceWork) work).getNumReduceTasks() == 1) { + LOG.info("Converting work to uber: {}", work); work.setUberMode(true); } }