diff --git itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniDruidCliDriver.java itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniDruidLlapCliDriver.java similarity index 94% rename from itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniDruidCliDriver.java rename to itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniDruidLlapCliDriver.java index fa75d65219..692879338e 100644 --- itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniDruidCliDriver.java +++ itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniDruidLlapCliDriver.java @@ -31,7 +31,8 @@ import java.util.List; @RunWith(Parameterized.class) -public class TestMiniDruidCliDriver { +public class TestMiniDruidLlapCliDriver +{ static CliAdapter adapter = new CliConfigs.MiniDruidCliConfig().getCliAdapter(); @@ -49,7 +50,7 @@ private String name; private File qfile; - public TestMiniDruidCliDriver(String name, File qfile) { + public TestMiniDruidLlapCliDriver(String name, File qfile) { this.name = name; this.qfile = qfile; } diff --git itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniDruidLlapLocalCliDriver.java itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniDruidLlapLocalCliDriver.java new file mode 100644 index 0000000000..7823e04310 --- /dev/null +++ itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniDruidLlapLocalCliDriver.java @@ -0,0 +1,46 @@ +package org.apache.hadoop.hive.cli; + +import org.apache.hadoop.hive.cli.control.CliAdapter; +import org.apache.hadoop.hive.cli.control.CliConfigs; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestRule; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.io.File; +import java.util.List; + +/** + * This is used only for dev debugging if needed + */ +@RunWith(Parameterized.class) +public class TestMiniDruidLlapLocalCliDriver { + static CliAdapter adapter = new CliConfigs.MiniDruidLlapLocalCliDriver().getCliAdapter(); + + @Parameterized.Parameters(name = "{0}") + public static List getParameters() throws Exception { + return adapter.getParameters(); + } + + @ClassRule + public static TestRule cliClassRule = adapter.buildClassRule(); + + @Rule + public TestRule cliTestRule = adapter.buildTestRule(); + + private String name; + private File qfile; + + public TestMiniDruidLlapLocalCliDriver(String name, File qfile) { + this.name = name; + this.qfile = qfile; + } + + @Test + public void testCliDriver() throws Exception { + adapter.runTest(name, qfile); + } + +} \ No newline at end of file diff --git itests/src/test/resources/testconfiguration.properties itests/src/test/resources/testconfiguration.properties index d08528f319..74c69b7984 100644 --- itests/src/test/resources/testconfiguration.properties +++ itests/src/test/resources/testconfiguration.properties @@ -1691,7 +1691,7 @@ spark.only.query.negative.files=spark_job_max_tasks.q,\ spark.perf.disabled.query.files=query14.q,\ query64.q -druid.query.files=druidmini_test1.q,\ +druid.llap.query.files=druidmini_test1.q,\ druid_basic2.q,\ druidmini_joins.q,\ druidmini_test_insert.q,\ @@ -1705,6 +1705,8 @@ druid.query.files=druidmini_test1.q,\ druidmini_floorTime.q, \ druidmini_masking.q +druid.llap.local.query.files=druidmini_noop.q + druid.kafka.query.files=druidkafkamini_basic.q # tests to be run by TestErasureCodingHDFSCliDriver and TestCliDriver diff --git itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java index d07599e96b..03b1b372bf 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java +++ itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java @@ -55,7 +55,7 @@ public CliConfig() { excludesFrom(testConfigProps, "miniSparkOnYarn.only.query.files"); excludesFrom(testConfigProps, "disabled.query.files"); excludesFrom(testConfigProps, "localSpark.only.query.files"); - excludesFrom(testConfigProps, "druid.query.files"); + excludesFrom(testConfigProps, "druid.llap.query.files"); excludesFrom(testConfigProps, "druid.kafka.query.files"); excludesFrom(testConfigProps, "erasurecoding.only.query.files"); @@ -174,7 +174,7 @@ public MiniDruidCliConfig() { try { setQueryDir("ql/src/test/queries/clientpositive"); - includesFrom(testConfigProps, "druid.query.files"); + includesFrom(testConfigProps, "druid.llap.query.files"); setResultsDir("ql/src/test/results/clientpositive/druid"); setLogDir("itests/qtest/target/tmp/log"); @@ -708,4 +708,33 @@ private void setHiveConfDir(MiniClusterType clusterType) { } } + public static class MiniDruidLlapLocalCliDriver extends AbstractCliConfig + { + + public MiniDruidLlapLocalCliDriver() + { + super(CoreCliDriver.class); + try { + setQueryDir("ql/src/test/queries/clientpositive"); + + includesFrom(testConfigProps, "druid.llap.local.query.files"); + + setResultsDir("ql/src/test/results/clientpositive/druid"); + setLogDir("itests/qtest/target/tmp/log"); + + setInitScript("q_test_druid_init.sql"); + setCleanupScript("q_test_cleanup_druid.sql"); + setHiveConfDir("data/conf/llap"); + setClusterType(MiniClusterType.druidLocal); + setMetastoreType(MetastoreType.sql); + setFsType(QTestUtil.FsType.local); + setQueryDir("ql/src/test/queries/clientpositive"); + + } + catch (Exception e) { + throw new RuntimeException("can't construct cliconfig", e); + } + } + } + } diff --git itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index 98aae5c118..4ed062e09e 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -422,6 +422,7 @@ private void createRemoteDirs() { llap_local(CoreClusterType.TEZ, FsType.local), none(CoreClusterType.MR, FsType.local), druid(CoreClusterType.TEZ, FsType.hdfs), + druidLocal(CoreClusterType.TEZ, FsType.local), druidKafka(CoreClusterType.TEZ, FsType.hdfs), kafka(CoreClusterType.TEZ, FsType.hdfs); @@ -460,6 +461,8 @@ public static MiniClusterType valueForString(String type) { return llap_local; } else if (type.equals("druid")) { return druid; + } else if (type.equals("druidLocal")) { + return druidLocal; } else if (type.equals("druid-kafka")) { return druidKafka; } @@ -632,7 +635,9 @@ private void setupMiniCluster(HadoopShims shims, String confDir) throws String uriString = fs.getUri().toString(); - if (clusterType == MiniClusterType.druid || clusterType == MiniClusterType.druidKafka) { + if (clusterType == MiniClusterType.druid + || clusterType == MiniClusterType.druidKafka + || clusterType == MiniClusterType.druidLocal) { final String tempDir = System.getProperty("test.tmp.dir"); druidCluster = new MiniDruidCluster("mini-druid", logDir, @@ -672,15 +677,28 @@ private void setupMiniCluster(HadoopShims shims, String confDir) throws + "/tez-site.xml")); } int numTrackers = 2; - if (EnumSet.of(MiniClusterType.llap, MiniClusterType.llap_local).contains(clusterType)) { + if (EnumSet.of( + MiniClusterType.llap, + MiniClusterType.llap_local, + MiniClusterType.druidLocal, + MiniClusterType.druid + ).contains(clusterType)) { llapCluster = LlapItUtils.startAndGetMiniLlapCluster(conf, setup.zooKeeperCluster, confDir); } else { } - if (EnumSet.of(MiniClusterType.llap_local, MiniClusterType.tez_local).contains(clusterType)) { - mr = shims.getLocalMiniTezCluster(conf, clusterType == MiniClusterType.llap_local); + if (EnumSet.of(MiniClusterType.llap_local, MiniClusterType.tez_local, MiniClusterType.druidLocal) + .contains(clusterType)) { + mr = shims.getLocalMiniTezCluster(conf, + clusterType == MiniClusterType.llap_local + || clusterType == MiniClusterType.druidLocal + ); } else { - mr = shims.getMiniTezCluster(conf, numTrackers, uriString, - EnumSet.of(MiniClusterType.llap, MiniClusterType.llap_local).contains(clusterType)); + mr = shims.getMiniTezCluster( + conf, + numTrackers, + uriString, + EnumSet.of(MiniClusterType.llap, MiniClusterType.llap_local, MiniClusterType.druid).contains(clusterType) + ); } } else if (clusterType == MiniClusterType.miniSparkOnYarn) { mr = shims.getMiniSparkCluster(conf, 2, uriString, 1); diff --git ql/src/test/results/clientpositive/druid/druid_basic2.q.out ql/src/test/results/clientpositive/druid/druid_basic2.q.out index 88916b9d6c..444d91db32 100644 --- ql/src/test/results/clientpositive/druid/druid_basic2.q.out +++ ql/src/test/results/clientpositive/druid/druid_basic2.q.out @@ -236,7 +236,8 @@ STAGE PLANS: tag: 0 value expressions: _col0 (type: string) auto parallelism: true - Execution mode: vectorized + Execution mode: vectorized, llap + LLAP IO: no inputs Path -> Alias: hdfs://### HDFS PATH ### [druid_table_1_n2] Path -> Partition: @@ -325,7 +326,8 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE tag: 1 auto parallelism: true - Execution mode: vectorized + Execution mode: vectorized, llap + LLAP IO: no inputs Path -> Alias: hdfs://### HDFS PATH ### [druid_table_1_n2] Path -> Partition: @@ -396,6 +398,7 @@ STAGE PLANS: Truncated Path -> Alias: /druid_table_1_n2 [druid_table_1_n2] Reducer 2 + Execution mode: llap Needs Tagging: false Reduce Operator Tree: Merge Join Operator @@ -501,7 +504,8 @@ STAGE PLANS: tag: 0 value expressions: _col0 (type: string) auto parallelism: false - Execution mode: vectorized + Execution mode: vectorized, llap + LLAP IO: no inputs Path -> Alias: hdfs://### HDFS PATH ### [druid_table_1_n2] Path -> Partition: @@ -588,7 +592,8 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE tag: 1 auto parallelism: false - Execution mode: vectorized + Execution mode: vectorized, llap + LLAP IO: no inputs Path -> Alias: hdfs://### HDFS PATH ### [druid_table_1_n2] Path -> Partition: @@ -659,6 +664,7 @@ STAGE PLANS: Truncated Path -> Alias: /druid_table_1_n2 [druid_table_1_n2] Reducer 2 + Execution mode: llap Needs Tagging: false Reduce Operator Tree: Merge Join Operator @@ -926,6 +932,8 @@ STAGE PLANS: tag: -1 value expressions: _col3 (type: float), _col4 (type: double) auto parallelism: true + Execution mode: llap + LLAP IO: no inputs Path -> Alias: hdfs://### HDFS PATH ### [druid_table_1_n2] Path -> Partition: @@ -988,6 +996,7 @@ STAGE PLANS: Truncated Path -> Alias: /druid_table_1_n2 [druid_table_1_n2] Reducer 2 + Execution mode: llap Needs Tagging: false Reduce Operator Tree: Group By Operator @@ -1011,6 +1020,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: timestamp with local time zone), _col3 (type: double) auto parallelism: false Reducer 3 + Execution mode: llap Needs Tagging: false Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/druid/druidmini_dynamic_partition.q.out ql/src/test/results/clientpositive/druid/druidmini_dynamic_partition.q.out index bd56df5088..cb2ae1d6a2 100644 --- ql/src/test/results/clientpositive/druid/druidmini_dynamic_partition.q.out +++ ql/src/test/results/clientpositive/druid/druidmini_dynamic_partition.q.out @@ -137,7 +137,10 @@ STAGE PLANS: Map-reduce partition columns: __time_granularity (type: timestamp), __druid_extra_partition_key (type: bigint) Statistics: Num rows: 12288 Data size: 2601650 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: timestamp with local time zone), _col1 (type: string), _col2 (type: string), _col3 (type: double), _col4 (type: float), _col5 (type: tinyint), _col6 (type: smallint), _col7 (type: int), _col8 (type: bigint), _col9 (type: boolean), _col10 (type: boolean) + Execution mode: llap + LLAP IO: all inputs Reducer 2 + Execution mode: llap Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: timestamp with local time zone), VALUE._col1 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: double), VALUE._col4 (type: float), VALUE._col5 (type: tinyint), VALUE._col6 (type: smallint), VALUE._col7 (type: int), VALUE._col8 (type: bigint), VALUE._col9 (type: boolean), VALUE._col10 (type: boolean), KEY.__time_granularity (type: timestamp), KEY.__druid_extra_partition_key (type: bigint) @@ -373,7 +376,10 @@ STAGE PLANS: Map-reduce partition columns: __time_granularity (type: timestamp), __druid_extra_partition_key (type: bigint) Statistics: Num rows: 12288 Data size: 2601650 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: timestamp with local time zone), _col1 (type: string), _col2 (type: string), _col3 (type: double), _col4 (type: float), _col5 (type: tinyint), _col6 (type: smallint), _col7 (type: int), _col8 (type: bigint), _col9 (type: boolean), _col10 (type: boolean) + Execution mode: llap + LLAP IO: all inputs Reducer 2 + Execution mode: llap Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: timestamp with local time zone), VALUE._col1 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: double), VALUE._col4 (type: float), VALUE._col5 (type: tinyint), VALUE._col6 (type: smallint), VALUE._col7 (type: int), VALUE._col8 (type: bigint), VALUE._col9 (type: boolean), VALUE._col10 (type: boolean), KEY.__time_granularity (type: timestamp), KEY.__druid_extra_partition_key (type: bigint) @@ -511,7 +517,10 @@ STAGE PLANS: Map-reduce partition columns: __time_granularity (type: timestamp), __druid_extra_partition_key (type: bigint) Statistics: Num rows: 12288 Data size: 2601650 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: timestamp with local time zone), _col1 (type: string), _col2 (type: string), _col3 (type: double), _col4 (type: float), _col5 (type: tinyint), _col6 (type: smallint), _col7 (type: int), _col8 (type: bigint), _col9 (type: boolean), _col10 (type: boolean) + Execution mode: llap + LLAP IO: all inputs Reducer 2 + Execution mode: llap Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: timestamp with local time zone), VALUE._col1 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: double), VALUE._col4 (type: float), VALUE._col5 (type: tinyint), VALUE._col6 (type: smallint), VALUE._col7 (type: int), VALUE._col8 (type: bigint), VALUE._col9 (type: boolean), VALUE._col10 (type: boolean), KEY.__time_granularity (type: timestamp), KEY.__druid_extra_partition_key (type: bigint) diff --git ql/src/test/results/clientpositive/druid/druidmini_expressions.q.out ql/src/test/results/clientpositive/druid/druidmini_expressions.q.out index e322d8f91f..610558ad49 100644 --- ql/src/test/results/clientpositive/druid/druidmini_expressions.q.out +++ ql/src/test/results/clientpositive/druid/druidmini_expressions.q.out @@ -260,6 +260,8 @@ STAGE PLANS: tag: -1 value expressions: _col0 (type: bigint) auto parallelism: false + Execution mode: llap + LLAP IO: no inputs Path -> Alias: hdfs://### HDFS PATH ### [druid_table_n0] Path -> Partition: @@ -334,6 +336,7 @@ STAGE PLANS: Truncated Path -> Alias: /druid_table_n0 [druid_table_n0] Reducer 2 + Execution mode: llap Needs Tagging: false Reduce Operator Tree: Group By Operator @@ -436,7 +439,10 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: timestamp with local time zone), _col1 (type: string) Statistics: Num rows: 9173 Data size: 3625856 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint), _col3 (type: double) + Execution mode: llap + LLAP IO: no inputs Reducer 2 + Execution mode: llap Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0), sum(VALUE._col1) @@ -504,7 +510,10 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: timestamp with local time zone), _col1 (type: string) Statistics: Num rows: 9173 Data size: 2091840 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint), _col3 (type: double) + Execution mode: llap + LLAP IO: no inputs Reducer 2 + Execution mode: llap Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0), sum(VALUE._col1) @@ -572,7 +581,10 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: timestamp with local time zone), _col1 (type: string) Statistics: Num rows: 9173 Data size: 3625856 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint), _col3 (type: double) + Execution mode: llap + LLAP IO: no inputs Reducer 2 + Execution mode: llap Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0), sum(VALUE._col1) @@ -640,7 +652,10 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: timestamp with local time zone), _col1 (type: string) Statistics: Num rows: 9173 Data size: 3625856 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint), _col3 (type: double) + Execution mode: llap + LLAP IO: no inputs Reducer 2 + Execution mode: llap Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0), sum(VALUE._col1) @@ -705,7 +720,10 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: no inputs Reducer 2 + Execution mode: llap Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -765,7 +783,10 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: double) + Execution mode: llap + LLAP IO: no inputs Reducer 2 + Execution mode: llap Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0), sum(VALUE._col1) @@ -825,7 +846,10 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: double), _col2 (type: int) + Execution mode: llap + LLAP IO: no inputs Reducer 2 + Execution mode: llap Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0), sum(VALUE._col1), min(VALUE._col2) @@ -1256,7 +1280,10 @@ STAGE PLANS: sort order: ++ Statistics: Num rows: 9173 Data size: 976192 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 + Execution mode: llap + LLAP IO: no inputs Reducer 2 + Execution mode: llap Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: date), KEY.reducesinkkey1 (type: date) diff --git ql/src/test/results/clientpositive/druid/druidmini_joins.q.out ql/src/test/results/clientpositive/druid/druidmini_joins.q.out index 014c7b5406..de0f2d8f21 100644 --- ql/src/test/results/clientpositive/druid/druidmini_joins.q.out +++ ql/src/test/results/clientpositive/druid/druidmini_joins.q.out @@ -117,6 +117,8 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 6 Data size: 1104 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Map 4 Map Operator Tree: TableScan @@ -133,7 +135,10 @@ STAGE PLANS: Map-reduce partition columns: username (type: string) Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: NONE value expressions: double1 (type: double) + Execution mode: llap + LLAP IO: no inputs Reducer 2 + Execution mode: llap Reduce Operator Tree: Merge Join Operator condition map: @@ -156,6 +161,7 @@ STAGE PLANS: Statistics: Num rows: 6 Data size: 1267 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: double) Reducer 3 + Execution mode: llap Reduce Operator Tree: Group By Operator aggregations: sum(VALUE._col0) @@ -220,5 +226,5 @@ POSTHOOK: Output: hdfs://### HDFS PATH ### alfred 10.300000190734863 bob 3.140000104904175 bonnie 0.0 -calvin 0.0 charlie 25.600000381469727 +calvin 0.0 diff --git ql/src/test/results/clientpositive/druid/druidmini_mv.q.out ql/src/test/results/clientpositive/druid/druidmini_mv.q.out deleted file mode 100644 index 54a4ef6d8c..0000000000 --- ql/src/test/results/clientpositive/druid/druidmini_mv.q.out +++ /dev/null @@ -1,661 +0,0 @@ -PREHOOK: query: CREATE TABLE cmv_basetable_n2 -STORED AS orc -TBLPROPERTIES ('transactional'='true') -AS -SELECT cast(current_timestamp() AS timestamp) AS t, - cast(a AS int) AS a, - cast(b AS varchar(256)) AS b, - cast(userid AS varchar(256)) AS userid, - cast(c AS double) AS c, - cast(d AS int) AS d -FROM TABLE ( - VALUES - (1, 'alfred', 'alfred', 10.30, 2), - (2, 'bob', 'bob', 3.14, 3), - (2, 'bonnie', 'bonnie', 172342.2, 3), - (3, 'calvin', 'calvin', 978.76, 3), - (3, 'charlie', 'charlie_a', 9.8, 1), - (3, 'charlie', 'charlie_b', 15.8, 1)) as q (a, b, userid, c, d) -PREHOOK: type: CREATETABLE_AS_SELECT -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: database:default -PREHOOK: Output: default@cmv_basetable_n2 -POSTHOOK: query: CREATE TABLE cmv_basetable_n2 -STORED AS orc -TBLPROPERTIES ('transactional'='true') -AS -SELECT cast(current_timestamp() AS timestamp) AS t, - cast(a AS int) AS a, - cast(b AS varchar(256)) AS b, - cast(userid AS varchar(256)) AS userid, - cast(c AS double) AS c, - cast(d AS int) AS d -FROM TABLE ( - VALUES - (1, 'alfred', 'alfred', 10.30, 2), - (2, 'bob', 'bob', 3.14, 3), - (2, 'bonnie', 'bonnie', 172342.2, 3), - (3, 'calvin', 'calvin', 978.76, 3), - (3, 'charlie', 'charlie_a', 9.8, 1), - (3, 'charlie', 'charlie_b', 15.8, 1)) as q (a, b, userid, c, d) -POSTHOOK: type: CREATETABLE_AS_SELECT -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: database:default -POSTHOOK: Output: default@cmv_basetable_n2 -POSTHOOK: Lineage: cmv_basetable_n2.a SCRIPT [] -POSTHOOK: Lineage: cmv_basetable_n2.b SCRIPT [] -POSTHOOK: Lineage: cmv_basetable_n2.c SCRIPT [] -POSTHOOK: Lineage: cmv_basetable_n2.d SCRIPT [] -POSTHOOK: Lineage: cmv_basetable_n2.t SIMPLE [] -POSTHOOK: Lineage: cmv_basetable_n2.userid SCRIPT [] -PREHOOK: query: CREATE MATERIALIZED VIEW cmv_mat_view_n2 -STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' -TBLPROPERTIES ("druid.segment.granularity" = "HOUR") -AS -SELECT cast(t AS timestamp with local time zone) as `__time`, a, b, c, userid -FROM cmv_basetable_n2 -WHERE a = 2 -PREHOOK: type: CREATE_MATERIALIZED_VIEW -PREHOOK: Input: default@cmv_basetable_n2 -PREHOOK: Output: database:default -PREHOOK: Output: default@cmv_mat_view_n2 -POSTHOOK: query: CREATE MATERIALIZED VIEW cmv_mat_view_n2 -STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' -TBLPROPERTIES ("druid.segment.granularity" = "HOUR") -AS -SELECT cast(t AS timestamp with local time zone) as `__time`, a, b, c, userid -FROM cmv_basetable_n2 -WHERE a = 2 -POSTHOOK: type: CREATE_MATERIALIZED_VIEW -POSTHOOK: Input: default@cmv_basetable_n2 -POSTHOOK: Output: database:default -POSTHOOK: Output: default@cmv_mat_view_n2 -PREHOOK: query: SELECT a, b, c FROM cmv_mat_view_n2 -PREHOOK: type: QUERY -PREHOOK: Input: default@cmv_mat_view_n2 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT a, b, c FROM cmv_mat_view_n2 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cmv_mat_view_n2 -POSTHOOK: Output: hdfs://### HDFS PATH ### -2 bob 3.140000104904175 -2 bonnie 172342.203125 -PREHOOK: query: SHOW TBLPROPERTIES cmv_mat_view_n2 -PREHOOK: type: SHOW_TBLPROPERTIES -POSTHOOK: query: SHOW TBLPROPERTIES cmv_mat_view_n2 -POSTHOOK: type: SHOW_TBLPROPERTIES -COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} -bucketing_version 2 -druid.datasource default.cmv_mat_view_n2 -druid.segment.granularity HOUR -numFiles 0 -numFilesErasureCoded 0 -numRows 2 -rawDataSize 0 -storage_handler org.apache.hadoop.hive.druid.DruidStorageHandler -totalSize 0 -#### A masked pattern was here #### -PREHOOK: query: CREATE MATERIALIZED VIEW IF NOT EXISTS cmv_mat_view2_n0 -STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' -TBLPROPERTIES ("druid.segment.granularity" = "HOUR") -AS -SELECT cast(t AS timestamp with local time zone) as `__time`, a, b, c, userid -FROM cmv_basetable_n2 -WHERE a = 3 -PREHOOK: type: CREATE_MATERIALIZED_VIEW -PREHOOK: Input: default@cmv_basetable_n2 -PREHOOK: Output: database:default -PREHOOK: Output: default@cmv_mat_view2_n0 -POSTHOOK: query: CREATE MATERIALIZED VIEW IF NOT EXISTS cmv_mat_view2_n0 -STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' -TBLPROPERTIES ("druid.segment.granularity" = "HOUR") -AS -SELECT cast(t AS timestamp with local time zone) as `__time`, a, b, c, userid -FROM cmv_basetable_n2 -WHERE a = 3 -POSTHOOK: type: CREATE_MATERIALIZED_VIEW -POSTHOOK: Input: default@cmv_basetable_n2 -POSTHOOK: Output: database:default -POSTHOOK: Output: default@cmv_mat_view2_n0 -PREHOOK: query: SELECT a, c FROM cmv_mat_view2_n0 -PREHOOK: type: QUERY -PREHOOK: Input: default@cmv_mat_view2_n0 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT a, c FROM cmv_mat_view2_n0 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cmv_mat_view2_n0 -POSTHOOK: Output: hdfs://### HDFS PATH ### -3 15.800000190734863 -3 9.800000190734863 -3 978.760009765625 -PREHOOK: query: SHOW TBLPROPERTIES cmv_mat_view2_n0 -PREHOOK: type: SHOW_TBLPROPERTIES -POSTHOOK: query: SHOW TBLPROPERTIES cmv_mat_view2_n0 -POSTHOOK: type: SHOW_TBLPROPERTIES -COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} -bucketing_version 2 -druid.datasource default.cmv_mat_view2_n0 -druid.segment.granularity HOUR -numFiles 0 -numFilesErasureCoded 0 -numRows 3 -rawDataSize 0 -storage_handler org.apache.hadoop.hive.druid.DruidStorageHandler -totalSize 0 -#### A masked pattern was here #### -PREHOOK: query: EXPLAIN -SELECT a, c -FROM cmv_basetable_n2 -WHERE a = 3 -PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN -SELECT a, c -FROM cmv_basetable_n2 -WHERE a = 3 -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: cmv_mat_view2_n0 - properties: - druid.fieldNames vc,c - druid.fieldTypes int,double - druid.query.json {"queryType":"scan","dataSource":"default.cmv_mat_view2_n0","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"virtualColumns":[{"type":"expression","name":"vc","expression":"3","outputType":"LONG"}],"columns":["vc","c"],"resultFormat":"compactedList"} - druid.query.type scan - Select Operator - expressions: vc (type: int), c (type: double) - outputColumnNames: _col0, _col1 - ListSink - -PREHOOK: query: SELECT a, c -FROM cmv_basetable_n2 -WHERE a = 3 -PREHOOK: type: QUERY -PREHOOK: Input: default@cmv_basetable_n2 -PREHOOK: Input: default@cmv_mat_view2_n0 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT a, c -FROM cmv_basetable_n2 -WHERE a = 3 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cmv_basetable_n2 -POSTHOOK: Input: default@cmv_mat_view2_n0 -POSTHOOK: Output: hdfs://### HDFS PATH ### -3 15.800000190734863 -3 9.800000190734863 -3 978.760009765625 -Warning: Shuffle Join MERGEJOIN[10][tables = [cmv_mat_view2_n0, $hdt$_0]] in Stage 'Reducer 2' is a cross product -PREHOOK: query: EXPLAIN -SELECT * FROM ( - (SELECT a, c FROM cmv_basetable_n2 WHERE a = 3) table1 - JOIN - (SELECT a, c FROM cmv_basetable_n2 WHERE d = 3) table2 - ON table1.a = table2.a) -PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN -SELECT * FROM ( - (SELECT a, c FROM cmv_basetable_n2 WHERE a = 3) table1 - JOIN - (SELECT a, c FROM cmv_basetable_n2 WHERE d = 3) table2 - ON table1.a = table2.a) -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (XPROD_EDGE), Map 3 (XPROD_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: cmv_basetable_n2 - filterExpr: ((a = 3) and (d = 3)) (type: boolean) - Statistics: Num rows: 6 Data size: 96 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((a = 3) and (d = 3)) (type: boolean) - Statistics: Num rows: 6 Data size: 96 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: c (type: double) - outputColumnNames: _col1 - Statistics: Num rows: 6 Data size: 96 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 6 Data size: 96 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: double) - Map 3 - Map Operator Tree: - TableScan - alias: cmv_mat_view2_n0 - properties: - druid.fieldNames a,c - druid.fieldTypes int,double - druid.query.json {"queryType":"scan","dataSource":"default.cmv_mat_view2_n0","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"columns":["a","c"],"resultFormat":"compactedList"} - druid.query.type scan - Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: NONE - value expressions: a (type: int), c (type: double) - Reducer 2 - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0, _col1, _col6 - Statistics: Num rows: 18 Data size: 522 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: double), _col0 (type: int), _col6 (type: double) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 18 Data size: 522 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 18 Data size: 522 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -Warning: Shuffle Join MERGEJOIN[10][tables = [cmv_mat_view2_n0, $hdt$_0]] in Stage 'Reducer 2' is a cross product -PREHOOK: query: SELECT * FROM ( - (SELECT a, c FROM cmv_basetable_n2 WHERE a = 3) table1 - JOIN - (SELECT a, c FROM cmv_basetable_n2 WHERE d = 3) table2 - ON table1.a = table2.a) -PREHOOK: type: QUERY -PREHOOK: Input: default@cmv_basetable_n2 -PREHOOK: Input: default@cmv_mat_view2_n0 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT * FROM ( - (SELECT a, c FROM cmv_basetable_n2 WHERE a = 3) table1 - JOIN - (SELECT a, c FROM cmv_basetable_n2 WHERE d = 3) table2 - ON table1.a = table2.a) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cmv_basetable_n2 -POSTHOOK: Input: default@cmv_mat_view2_n0 -POSTHOOK: Output: hdfs://### HDFS PATH ### -3 15.800000190734863 3 978.76 -3 9.800000190734863 3 978.76 -3 978.760009765625 3 978.76 -PREHOOK: query: INSERT INTO cmv_basetable_n2 VALUES - (cast(current_timestamp() AS timestamp), 3, 'charlie', 'charlie_c', 15.8, 1) -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: default@cmv_basetable_n2 -POSTHOOK: query: INSERT INTO cmv_basetable_n2 VALUES - (cast(current_timestamp() AS timestamp), 3, 'charlie', 'charlie_c', 15.8, 1) -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: default@cmv_basetable_n2 -POSTHOOK: Lineage: cmv_basetable_n2.a SCRIPT [] -POSTHOOK: Lineage: cmv_basetable_n2.b SCRIPT [] -POSTHOOK: Lineage: cmv_basetable_n2.c SCRIPT [] -POSTHOOK: Lineage: cmv_basetable_n2.d SCRIPT [] -POSTHOOK: Lineage: cmv_basetable_n2.t SCRIPT [] -POSTHOOK: Lineage: cmv_basetable_n2.userid SCRIPT [] -Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product -PREHOOK: query: EXPLAIN -SELECT * FROM ( - (SELECT a, c FROM cmv_basetable_n2 WHERE a = 3) table1 - JOIN - (SELECT a, c FROM cmv_basetable_n2 WHERE d = 3) table2 - ON table1.a = table2.a) -PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN -SELECT * FROM ( - (SELECT a, c FROM cmv_basetable_n2 WHERE a = 3) table1 - JOIN - (SELECT a, c FROM cmv_basetable_n2 WHERE d = 3) table2 - ON table1.a = table2.a) -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (XPROD_EDGE), Map 3 (XPROD_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: cmv_basetable_n2 - filterExpr: (a = 3) (type: boolean) - Statistics: Num rows: 7 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (a = 3) (type: boolean) - Statistics: Num rows: 7 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: c (type: double) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 7 Data size: 84 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: double) - Map 3 - Map Operator Tree: - TableScan - alias: cmv_basetable_n2 - filterExpr: ((d = 3) and (a = 3)) (type: boolean) - Statistics: Num rows: 7 Data size: 112 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((a = 3) and (d = 3)) (type: boolean) - Statistics: Num rows: 7 Data size: 112 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: c (type: double) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 112 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 7 Data size: 112 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: double) - Reducer 2 - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0, _col1 - Statistics: Num rows: 49 Data size: 1421 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: 3 (type: int), _col0 (type: double), 3 (type: int), _col1 (type: double) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 49 Data size: 1421 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 49 Data size: 1421 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product -PREHOOK: query: SELECT * FROM ( - (SELECT a, c FROM cmv_basetable_n2 WHERE a = 3) table1 - JOIN - (SELECT a, c FROM cmv_basetable_n2 WHERE d = 3) table2 - ON table1.a = table2.a) -PREHOOK: type: QUERY -PREHOOK: Input: default@cmv_basetable_n2 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT * FROM ( - (SELECT a, c FROM cmv_basetable_n2 WHERE a = 3) table1 - JOIN - (SELECT a, c FROM cmv_basetable_n2 WHERE d = 3) table2 - ON table1.a = table2.a) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cmv_basetable_n2 -POSTHOOK: Output: hdfs://### HDFS PATH ### -3 15.8 3 978.76 -3 15.8 3 978.76 -3 9.8 3 978.76 -3 978.76 3 978.76 -PREHOOK: query: EXPLAIN -ALTER MATERIALIZED VIEW cmv_mat_view2_n0 REBUILD -PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN -ALTER MATERIALIZED VIEW cmv_mat_view2_n0 REBUILD -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-0 is a root stage - Stage-4 depends on stages: Stage-0, Stage-1, Stage-2 - Stage-1 is a root stage - Stage-2 is a root stage - Stage-5 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-0 - Alter Table Operator: - Alter Table - type: drop props - old name: default.cmv_mat_view2_n0 - properties: - COLUMN_STATS_ACCURATE - - Stage: Stage-4 - Materialized View Work - - Stage: Stage-1 - Pre Insert operator: - Pre-Insert task - - Stage: Stage-2 - Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: cmv_basetable_n2 - filterExpr: (a = 3) (type: boolean) - Statistics: Num rows: 7 Data size: 5124 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (a = 3) (type: boolean) - Statistics: Num rows: 7 Data size: 5124 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: CAST( t AS timestamp with local time zone) (type: timestamp with local time zone), 3 (type: int), b (type: varchar(256)), c (type: double), userid (type: varchar(256)) - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 7 Data size: 5124 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: timestamp with local time zone), _col1 (type: int), _col2 (type: varchar(256)), _col3 (type: double), _col4 (type: varchar(256)), floor_hour(CAST( GenericUDFEpochMilli(_col0) AS TIMESTAMP)) (type: timestamp) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, __time_granularity - Statistics: Num rows: 7 Data size: 5124 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: __time_granularity (type: timestamp) - sort order: + - Map-reduce partition columns: __time_granularity (type: timestamp) - Statistics: Num rows: 7 Data size: 5124 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: timestamp with local time zone), _col1 (type: int), _col2 (type: varchar(256)), _col3 (type: double), _col4 (type: varchar(256)) - Reducer 2 - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: timestamp with local time zone), VALUE._col1 (type: int), VALUE._col2 (type: varchar(256)), VALUE._col3 (type: double), VALUE._col4 (type: varchar(256)), KEY.__time_granularity (type: timestamp) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, __time_granularity - Statistics: Num rows: 7 Data size: 5124 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 7 Data size: 5124 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.druid.io.DruidQueryBasedInputFormat - output format: org.apache.hadoop.hive.druid.io.DruidOutputFormat - serde: org.apache.hadoop.hive.druid.serde.DruidSerDe - name: default.cmv_mat_view2_n0 - - Stage: Stage-5 - Insert operator: - Commit-Insert-Hook - -PREHOOK: query: ALTER MATERIALIZED VIEW cmv_mat_view2_n0 REBUILD -PREHOOK: type: QUERY -PREHOOK: Input: default@cmv_basetable_n2 -PREHOOK: Output: default@cmv_mat_view2_n0 -POSTHOOK: query: ALTER MATERIALIZED VIEW cmv_mat_view2_n0 REBUILD -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cmv_basetable_n2 -POSTHOOK: Output: default@cmv_mat_view2_n0 -PREHOOK: query: SHOW TBLPROPERTIES cmv_mat_view2_n0 -PREHOOK: type: SHOW_TBLPROPERTIES -POSTHOOK: query: SHOW TBLPROPERTIES cmv_mat_view2_n0 -POSTHOOK: type: SHOW_TBLPROPERTIES -bucketing_version 2 -druid.datasource default.cmv_mat_view2_n0 -druid.segment.granularity HOUR -#### A masked pattern was here #### -numFiles 0 -numFilesErasureCoded 0 -numRows 3 -rawDataSize 0 -storage_handler org.apache.hadoop.hive.druid.DruidStorageHandler -totalSize 0 -#### A masked pattern was here #### -Warning: Shuffle Join MERGEJOIN[10][tables = [cmv_mat_view2_n0, $hdt$_0]] in Stage 'Reducer 2' is a cross product -PREHOOK: query: EXPLAIN -SELECT * FROM ( - (SELECT a, c FROM cmv_basetable_n2 WHERE a = 3) table1 - JOIN - (SELECT a, c FROM cmv_basetable_n2 WHERE d = 3) table2 - ON table1.a = table2.a) -PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN -SELECT * FROM ( - (SELECT a, c FROM cmv_basetable_n2 WHERE a = 3) table1 - JOIN - (SELECT a, c FROM cmv_basetable_n2 WHERE d = 3) table2 - ON table1.a = table2.a) -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (XPROD_EDGE), Map 3 (XPROD_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: cmv_basetable_n2 - filterExpr: ((a = 3) and (d = 3)) (type: boolean) - Statistics: Num rows: 7 Data size: 112 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((a = 3) and (d = 3)) (type: boolean) - Statistics: Num rows: 7 Data size: 112 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: c (type: double) - outputColumnNames: _col1 - Statistics: Num rows: 7 Data size: 112 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 7 Data size: 112 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: double) - Map 3 - Map Operator Tree: - TableScan - alias: cmv_mat_view2_n0 - properties: - druid.fieldNames a,c - druid.fieldTypes int,double - druid.query.json {"queryType":"scan","dataSource":"default.cmv_mat_view2_n0","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"columns":["a","c"],"resultFormat":"compactedList"} - druid.query.type scan - Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: NONE - value expressions: a (type: int), c (type: double) - Reducer 2 - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0, _col1, _col6 - Statistics: Num rows: 21 Data size: 609 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: double), _col0 (type: int), _col6 (type: double) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 21 Data size: 609 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 21 Data size: 609 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -Warning: Shuffle Join MERGEJOIN[10][tables = [cmv_mat_view2_n0, $hdt$_0]] in Stage 'Reducer 2' is a cross product -PREHOOK: query: SELECT * FROM ( - (SELECT a, c FROM cmv_basetable_n2 WHERE a = 3) table1 - JOIN - (SELECT a, c FROM cmv_basetable_n2 WHERE d = 3) table2 - ON table1.a = table2.a) -PREHOOK: type: QUERY -PREHOOK: Input: default@cmv_basetable_n2 -PREHOOK: Input: default@cmv_mat_view2_n0 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT * FROM ( - (SELECT a, c FROM cmv_basetable_n2 WHERE a = 3) table1 - JOIN - (SELECT a, c FROM cmv_basetable_n2 WHERE d = 3) table2 - ON table1.a = table2.a) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cmv_basetable_n2 -POSTHOOK: Input: default@cmv_mat_view2_n0 -POSTHOOK: Output: hdfs://### HDFS PATH ### -3 15.800000190734863 3 978.76 -3 15.800000190734863 3 978.76 -3 9.800000190734863 3 978.76 -3 978.760009765625 3 978.76 -PREHOOK: query: DROP MATERIALIZED VIEW cmv_mat_view_n2 -PREHOOK: type: DROP_MATERIALIZED_VIEW -PREHOOK: Input: default@cmv_mat_view_n2 -PREHOOK: Output: default@cmv_mat_view_n2 -POSTHOOK: query: DROP MATERIALIZED VIEW cmv_mat_view_n2 -POSTHOOK: type: DROP_MATERIALIZED_VIEW -POSTHOOK: Input: default@cmv_mat_view_n2 -POSTHOOK: Output: default@cmv_mat_view_n2 -PREHOOK: query: DROP MATERIALIZED VIEW cmv_mat_view2_n0 -PREHOOK: type: DROP_MATERIALIZED_VIEW -PREHOOK: Input: default@cmv_mat_view2_n0 -PREHOOK: Output: default@cmv_mat_view2_n0 -POSTHOOK: query: DROP MATERIALIZED VIEW cmv_mat_view2_n0 -POSTHOOK: type: DROP_MATERIALIZED_VIEW -POSTHOOK: Input: default@cmv_mat_view2_n0 -POSTHOOK: Output: default@cmv_mat_view2_n0 -PREHOOK: query: DROP TABLE cmv_basetable_n2 -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@cmv_basetable_n2 -PREHOOK: Output: default@cmv_basetable_n2 -POSTHOOK: query: DROP TABLE cmv_basetable_n2 -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@cmv_basetable_n2 -POSTHOOK: Output: default@cmv_basetable_n2 diff --git ql/src/test/results/clientpositive/druid/druidmini_test1.q.out ql/src/test/results/clientpositive/druid/druidmini_test1.q.out index 7f6c6b0aa3..f93841f11c 100644 --- ql/src/test/results/clientpositive/druid/druidmini_test1.q.out +++ ql/src/test/results/clientpositive/druid/druidmini_test1.q.out @@ -316,7 +316,10 @@ STAGE PLANS: sort order: + Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 + Execution mode: llap + LLAP IO: no inputs Reducer 2 + Execution mode: llap Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: timestamp with local time zone) @@ -400,7 +403,10 @@ STAGE PLANS: sort order: + Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 + Execution mode: llap + LLAP IO: no inputs Reducer 2 + Execution mode: llap Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: timestamp with local time zone) @@ -486,7 +492,10 @@ STAGE PLANS: sort order: + Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 + Execution mode: llap + LLAP IO: no inputs Reducer 2 + Execution mode: llap Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: timestamp with local time zone) @@ -574,7 +583,10 @@ STAGE PLANS: sort order: + Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 + Execution mode: llap + LLAP IO: no inputs Reducer 2 + Execution mode: llap Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: timestamp with local time zone) @@ -662,7 +674,10 @@ STAGE PLANS: sort order: + Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 + Execution mode: llap + LLAP IO: no inputs Reducer 2 + Execution mode: llap Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: timestamp with local time zone) @@ -750,7 +765,10 @@ STAGE PLANS: sort order: + Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 + Execution mode: llap + LLAP IO: no inputs Reducer 2 + Execution mode: llap Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: timestamp with local time zone) diff --git testutils/ptest2/conf/deployed/master-mr2.properties testutils/ptest2/conf/deployed/master-mr2.properties index 90a654cf8a..fff5273062 100644 --- testutils/ptest2/conf/deployed/master-mr2.properties +++ testutils/ptest2/conf/deployed/master-mr2.properties @@ -180,9 +180,9 @@ qFileTest.erasurecodingCli.queryFilesProperty = qfile qFileTest.erasurecodingCli.include = normal qFileTest.erasurecodingCli.groups.normal = mainProperties.${erasurecoding.only.query.files} mainProperties.${erasurecoding.shared.query.files} -qFileTest.miniDruid.driver = TestMiniDruidCliDriver +qFileTest.miniDruid.driver = TestMiniDruidLlapCliDriver qFileTest.miniDruid.directory = ql/src/test/queries/clientpositive qFileTest.miniDruid.batchSize = 5 qFileTest.miniDruid.queryFilesProperty = qfile qFileTest.miniDruid.include = normal -qFileTest.miniDruid.groups.normal = mainProperties.${druid.query.files} +qFileTest.miniDruid.groups.normal = mainProperties.${druid.llap.query.files}